Merge pull request #35 from lnyemba/v2.2.0

V2.2.0

bin/transport

@@ -177,7 +177,25 @@ def register (label:Annotated[str,typer.Argument(help="unique label that will be
     except Exception as e:
         _msg = f"""{TIMES_MARK} {e}"""
     print (_msg)
-    
+@app_r.command(name="template")    
+def template(name:Annotated[str,typer.Argument(help="database technology provider" ) ]):
+    """
+    This function will generate a template entry for the registry (content of an auth file)
+    """
+    #
+    # retrieve the provider and display the template if it has one
+    for _module in  ['sql','cloud','warehouse','nosql','other'] :
+        ref = getattr(transport,_module) if hasattr(transport,_module) else None
+        _entry = {}
+        if ref :
+            if hasattr(ref,name) :
+                _pointer = getattr(ref,name)
+                _entry = dict({'provider':name},**_pointer.template()) if hasattr(_pointer,'template') else {}
+                break
+    #
+    #
+    print ( json.dumps(_entry))
+    pass
 @app_r.command(name="list")
 def register_list ():
     """

transport/cloud/bigquery.py

@@ -14,6 +14,8 @@ import numpy as np
 import time
 
 MAX_CHUNK = 2000000
+def template ():
+    return {'provider':'bigquery','private_key':'path-to-key','dataset':'name-of-dataset','table':'table'}
 class BigQuery:
     def __init__(self,**_args):
         path = _args['service_key'] if 'service_key' in _args else _args['private_key']
@@ -23,6 +25,7 @@ class BigQuery:
         self.dtypes = _args['dtypes'] if 'dtypes' in _args else None
         self.table = _args['table'] if 'table' in _args else None
         self.client = bq.Client.from_service_account_json(self.path)
+
     def meta(self,**_args):
         """
         This function returns meta data for a given table or query with dataset/table properly formatted

transport/cloud/databricks.py

@@ -17,6 +17,8 @@ import sqlalchemy
 # from transport.common import Reader,Writer
 import pandas as pd
 
+def template ():
+    return {'provider':'databricks','host':'fqn-host','token':'token','cluster_path':'path-of-cluster','catalog':'name-of-catalog','database':'schema-or-database','table':'table'}
 
 class Bricks:
     """

transport/cloud/nextcloud.py

@@ -8,6 +8,8 @@ import pandas as pd
 from io import StringIO
 import json
 import nextcloud_client as nextcloud
+def template():
+    return {"url":None,"token":None,"uid":None,"file":None}
 
 class Nextcloud :
     def __init__(self,**_args):

transport/cloud/s3.py

@@ -20,6 +20,8 @@ from io import StringIO
 import pandas as pd
 import json
 
+def template():
+	return {'access_key':'access-key','secret_key':'secret-key','region':'region','bucket':'name-of-bucket','file':'file-name','chunksize':10000}
 class s3 :
 	"""
 		@TODO: Implement a search function for a file given a bucket??

transport/info.py

@@ -1,6 +1,6 @@
 __app_name__  = 'data-transport'
 __author__ = 'The Phi Technology'
-__version__= '2.2.30'
+__version__= '2.2.40'
 __email__  = "info@the-phi.com"
 __edition__= 'community'
 __license__=f"""

transport/nosql/couchdb.py

@@ -11,6 +11,8 @@ import sys
 # from transport.common import Reader, Writer
 from datetime import datetime
 
+def template():
+	return {'dbname':'database','doc':'document','username':'username','password':'password','url':'url-with-port'}
 
 class Couch:
 	"""

transport/nosql/mongodb.py

@@ -19,7 +19,8 @@ import json
 import re
 from multiprocessing import Lock, RLock
 from transport.common import IEncoder
-
+def template():
+    return {'provider':'mongodb','host':'localhost','port':27017,'db':'db-name','collection':'collection-name','username':'username','password':'password','mechanism':'SCRAM-SHA-256'}
 class Mongo :
     lock = RLock()
     """

transport/other/files.py

@@ -4,6 +4,10 @@ This file is a wrapper around pandas built-in functionalities to handle characte
 import pandas as pd
 import numpy as np
 import os
+
+def template():
+	return {'path':None,'delimiter':None}
+
 class File :	
 	def __init__(self,**params):
 		"""

transport/other/http.py

@@ -7,7 +7,8 @@ import requests
 from io import StringIO
 import pandas as pd
 
-
+def template():
+	return {'url':None, 'headers':{'key':'value'}}
 class Reader:
 	"""
 	This class is designed to read data from an Http request file handler provided to us by flask

transport/other/rabbitmq.py

@@ -17,6 +17,10 @@ import sys
 # 	from common import Reader, Writer
 import json
 from multiprocessing import RLock
+
+def template():
+	return {'port':5672,'host':'localhost','queue':None,'vhost':None,'username':None,'password':None}
+
 class MessageQueue:
 	"""
 		This class hierarchy is designed to handle interactions with a queue server using pika framework (our tests are based on rabbitmq)

transport/sql/common.py

@@ -6,7 +6,8 @@ import sqlalchemy as sqa
 from sqlalchemy import text , MetaData, inspect
 
 import pandas as pd
-
+def template():
+    return {'host':'localhost','database':'database','table':'table'}
 class Base:
     def __init__(self,**_args):
         self._host = _args['host'] if 'host' in _args else 'localhost'

transport/sql/duckdb.py

@@ -2,7 +2,8 @@
 This module implements the handler for duckdb (in memory or not)
 """
 from transport.sql.common import Base, BaseReader, BaseWriter
-
+def template ():
+    return {'database':'path-to-database','table':'table'}
 class Duck :
     def __init__(self,**_args):
         #

transport/sql/mysql.py

@@ -1,8 +1,13 @@
 """
 This file implements support for mysql and maria db (with drivers mysql+mysql)
 """
-from transport.sql.common import BaseReader, BaseWriter
+from transport.sql.common import BaseReader, BaseWriter, template as _template
+# import transport.sql.common.template as _template
+
 # import mysql.connector as my
+def template ():
+    return dict(_template(),**{'port':3306})
+
 class MYSQL:
     
     def get_provider(self):

transport/sql/netezza.py

@@ -1,6 +1,9 @@
 import nzpy as nz   
-from transport.sql.common import BaseReader, BaseWriter
+from transport.sql.common import BaseReader, BaseWriter, template as _template
 
+
+def template ():
+    return dict(_template(),**{'port':5480})
 class Netezza:
     def get_provider(self):
         return 'netezza+nzpy'

transport/sql/postgresql.py

@@ -1,10 +1,13 @@
 
-from transport.sql.common import BaseReader , BaseWriter
+from transport.sql.common import BaseReader , BaseWriter, template as _template
 from psycopg2.extensions import register_adapter, AsIs
 import numpy as np
 
 register_adapter(np.int64, AsIs)
 
+def template ():
+    return dict(_template(),**{'port':5432,'chunksize':10000})
+
 class PG:
     def __init__(self,**_args):
         super().__init__(**_args)

transport/sql/sqlite.py

@@ -1,6 +1,8 @@
 import sqlalchemy
 import pandas as pd
 from transport.sql.common import Base, BaseReader, BaseWriter
+def template():
+    return {'database':'path-to-database','table':'table'}
 class SQLite (BaseReader):
     def __init__(self,**_args):
         super().__init__(**_args)

transport/sql/sqlserver.py

@@ -3,7 +3,11 @@ Handling Microsoft SQL Server via pymssql driver/connector
 """
 import sqlalchemy
 import pandas as pd
-from transport.sql.common import Base, BaseReader, BaseWriter
+from transport.sql.common import Base, BaseReader, BaseWriter, template as _template
+
+
+def template ():
+    return dict(_template(),**{'port':1433})
 
 
 class MsSQLServer:

transport/warehouse/drill.py

@@ -3,6 +3,10 @@ import pandas as pd
 from .. sql.common import BaseReader , BaseWriter
 import sqlalchemy as sqa
 
+
+def template():
+    return {'host':'localhost','port':8047,'ssl':False,'table':None,'database':None}
+
 class Drill :
     __template = {'host':None,'port':None,'ssl':None,'table':None,'database':None}
     def __init__(self,**_args):

transport/warehouse/iceberg.py

@@ -11,6 +11,8 @@ from pyspark.sql.types import *
 from pyspark.sql.functions import col, to_date, to_timestamp
 import copy
 
+def template():
+    return {'catalog':None,'database':None,'table':None}
 class Iceberg :
     def __init__(self,**_args):
         """