Add column description ingestion to sample data loader (#123)

example/models/test_column_model.py

@@ -4,7 +4,7 @@ from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
 
-from databuilder.models.table_metadata import TableMetadata
+from databuilder.models.table_metadata import TableMetadata, DESCRIPTION_NODE_LABEL
 
 
 class TestColumnMetadata(Neo4jCsvSerializable):
@@ -76,6 +76,14 @@ class TestColumnMetadata(Neo4jCsvSerializable):
                                                            tbl=self.table_name,
                                                            col=self.name)
 
+    def _get_col_description_key(self):
+        # type: (TestColumnMetadata) -> str
+        return TestColumnMetadata.COLUMN_DESCRIPTION_FORMAT.format(db=self.database,
+                                                                   cluster=self.cluster,
+                                                                   schema=self.schema_name,
+                                                                   tbl=self.table_name,
+                                                                   col=self.name)
+
     def _get_table_key(self):
         # type: () -> str
         return TableMetadata.TABLE_KEY_FORMAT.format(db=self.database,
@@ -89,15 +97,21 @@ class TestColumnMetadata(Neo4jCsvSerializable):
         Create a list of Neo4j node records
         :return:
         """
-        results = []
-
-        results.append({
+        results = [{
             NODE_LABEL: TestColumnMetadata.COLUMN_NODE_LABEL,
             NODE_KEY: self._get_col_key(),
             TestColumnMetadata.COLUMN_NAME: self.name,
             TestColumnMetadata.COLUMN_TYPE: self.type,
             TestColumnMetadata.COLUMN_ORDER: self.sort_order
+        }]
+
+        if self.description:
+            results.append({
+                NODE_LABEL: DESCRIPTION_NODE_LABEL,
+                NODE_KEY: self._get_col_description_key(),
+                TestColumnMetadata.COLUMN_DESCRIPTION: self.description
             })
+
         return results
 
     def create_relation(self):
@@ -106,7 +120,8 @@ class TestColumnMetadata(Neo4jCsvSerializable):
         Create a list of relation map between watermark record with original hive table
         :return:
         """
-        return [{
+
+        results = [{
             RELATION_START_LABEL: TableMetadata.TABLE_NODE_LABEL,
             RELATION_END_LABEL: TestColumnMetadata.COLUMN_NODE_LABEL,
             RELATION_START_KEY: self._get_table_key(),
@@ -115,6 +130,18 @@ class TestColumnMetadata(Neo4jCsvSerializable):
             RELATION_REVERSE_TYPE: TableMetadata.COL_TABLE_RELATION_TYPE
         }]
 
+        if self.description:
+            results.append({
+                RELATION_START_LABEL: self.COLUMN_NODE_LABEL,
+                RELATION_END_LABEL: self.DESCRIPTION_NODE_LABEL,
+                RELATION_START_KEY: self._get_col_key(),
+                RELATION_END_KEY: self._get_col_description_key(),
+                RELATION_TYPE: self.COL_DESCRIPTION_RELATION_TYPE,
+                RELATION_REVERSE_TYPE: self.DESCRIPTION_COL_RELATION_TYPE
+            })
+
+        return results
+
     def __repr__(self):
         # type: () -> str
         return 'ColumnMetadata({!r}, {!r}, {!r}, {!r})'.format(self.name,

example/sample_data/sample_col.csv

 name,description,col_type,sort_order,database,cluster,schema_name,table_name,table_desc
-col1,"col1","string",1,hive,gold,test_schema,test_table1,"1st test table"
-col2,"col2","string",2,hive,gold,test_schema,test_table1,"1st test table"
-col3,"col3","string",3,hive,gold,test_schema,test_table1,"1st test table"
-col4,"col4","string",4,hive,gold,test_schema,test_table1,"1st test table"
-col1,"col1","string",1,dynamo,gold,test_schema,test_table2,"2nd test table"
-col2,"col2","string",2,dynamo,gold,test_schema,test_table2,"2nd test table"
-col3,"col3","string",3,dynamo,gold,test_schema,test_table2,"2nd test table"
-col4,"col4","string",4,dynamo,gold,test_schema,test_table2,"2nd test table"
\ No newline at end of file
+col1,"col1 description","string",1,hive,gold,test_schema,test_table1,"1st test table"
+col2,"col2 description","string",2,hive,gold,test_schema,test_table1,"1st test table"
+col3,"col3 description","string",3,hive,gold,test_schema,test_table1,"1st test table"
+col4,"col4 description","string",4,hive,gold,test_schema,test_table1,"1st test table"
+col1,"col1 description","string",1,dynamo,gold,test_schema,test_table2,"2nd test table"
+col2,"col2 description","string",2,dynamo,gold,test_schema,test_table2,"2nd test table"
+col3,"col3 description","string",3,dynamo,gold,test_schema,test_table2,"2nd test table"
+col4,"col4 description","string",4,dynamo,gold,test_schema,test_table2,"2nd test table"
\ No newline at end of file