Avoid using ConfigTree.with_fallback method in job level

databuilder/job/job.py

@@ -48,8 +48,7 @@ class DefaultJob(Job):
 
     def _init(self):
         # type: () -> None
-        task_conf = Scoped.get_scoped_conf(self.conf, self.task.get_scope())
-        self.task.init(task_conf.with_fallback(self.conf))
+        self.task.init(self.conf)
 
     def launch(self):
         # type: () -> None

databuilder/task/neo4j_staleness_removal_task.py

@@ -6,9 +6,11 @@ from pyhocon import ConfigFactory  # noqa: F401
 from pyhocon import ConfigTree  # noqa: F401
 from typing import Dict, Iterable, Any  # noqa: F401
 
+from databuilder import Scoped
 from databuilder.task.base_task import Task  # noqa: F401
 from databuilder.publisher.neo4j_csv_publisher import JOB_PUBLISH_TAG
 
+
 # A end point for Neo4j e.g: bolt://localhost:9999
 NEO4J_END_POINT_KEY = 'neo4j_endpoint'
 NEO4J_MAX_CONN_LIFE_TIME_SEC = 'neo4j_max_conn_life_time_sec'
@@ -53,8 +55,9 @@ class Neo4jStalenessRemovalTask(Task):
 
     def init(self, conf):
         # type: (ConfigTree) -> None
-
-        conf = conf.with_fallback(DEFAULT_CONFIG)
+        conf = Scoped.get_scoped_conf(conf, self.get_scope())\
+            .with_fallback(conf)\
+            .with_fallback(DEFAULT_CONFIG)
         self.target_nodes = set(conf.get_list(TARGET_NODES))
         self.target_relations = set(conf.get_list(TARGET_RELATIONS))
         self.batch_size = conf.get_int(BATCH_SIZE)

setup.py

 from setuptools import setup, find_packages
 
 
-__version__ = '1.0.3'
+__version__ = '1.0.4'
 
 setup(
     name='amundsen-databuilder',

tests/unit/task/test_neo4j_staleness_removal_task.py

@@ -23,13 +23,13 @@ class TestRemoveStaleData(unittest.TestCase):
             task = Neo4jStalenessRemovalTask()
             job_config = ConfigFactory.from_dict({
                 'job.identifier': 'remove_stale_data_job',
-                neo4j_staleness_removal_task.NEO4J_END_POINT_KEY:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
                     'foobar',
-                neo4j_staleness_removal_task.NEO4J_USER:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
                     'foo',
-                neo4j_staleness_removal_task.NEO4J_PASSWORD:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
                     'bar',
-                neo4j_staleness_removal_task.STALENESS_MAX_PCT:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
                     90,
                 neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo'
             })
@@ -47,13 +47,13 @@ class TestRemoveStaleData(unittest.TestCase):
             task = Neo4jStalenessRemovalTask()
             job_config = ConfigFactory.from_dict({
                 'job.identifier': 'remove_stale_data_job',
-                neo4j_staleness_removal_task.NEO4J_END_POINT_KEY:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
                     'foobar',
-                neo4j_staleness_removal_task.NEO4J_USER:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
                     'foo',
-                neo4j_staleness_removal_task.NEO4J_PASSWORD:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
                     'bar',
-                neo4j_staleness_removal_task.STALENESS_MAX_PCT:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
                     5,
                 neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo'
             })
@@ -71,15 +71,15 @@ class TestRemoveStaleData(unittest.TestCase):
             task = Neo4jStalenessRemovalTask()
             job_config = ConfigFactory.from_dict({
                 'job.identifier': 'remove_stale_data_job',
-                neo4j_staleness_removal_task.NEO4J_END_POINT_KEY:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
                     'foobar',
-                neo4j_staleness_removal_task.NEO4J_USER:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
                     'foo',
-                neo4j_staleness_removal_task.NEO4J_PASSWORD:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
                     'bar',
-                neo4j_staleness_removal_task.STALENESS_MAX_PCT:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
                     5,
-                neo4j_staleness_removal_task.STALENESS_PCT_MAX_DICT:
+                '{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_PCT_MAX_DICT):
                     {'foo': 51},
                 neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo'
             })