Browse Source

bug fix mongodb,bigquery

Steve Nyemba 2 years ago
parent
commit
b8fb538ec7
3 changed files with 9 additions and 6 deletions
  1. 1 1
      setup.py
  2. 5 5
      transport/mongo.py
  3. 3 0
      transport/sql.py

+ 1 - 1
setup.py

@@ -8,7 +8,7 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
 args    = {
     "name":"data-transport",
-    "version":"1.6.0",
+    "version":"1.6.1",
     "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
     "license":"MIT",
     "packages":["transport"]}

+ 5 - 5
transport/mongo.py

@@ -34,7 +34,7 @@ class Mongo :
             :password   password for current user
         """
 
-        self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
+        self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
         # authSource=(args['authSource'] if 'authSource' in args else self.dbname)
         self._lock = False if 'lock' not in args else args['lock']
 
@@ -64,7 +64,7 @@ class Mongo :
                       username=username,
                       password=password ,
                       authSource=self.authSource,
-                      authMechanism=self.authMechanism)
+                      authMechanism=self.mechanism)
             
         else:
             self.client = MongoClient(self.host,maxPoolSize=10000)                    
@@ -76,7 +76,7 @@ class Mongo :
         q = self.uid in self.client[self.dbname].list_collection_names()
         return p and q
     def setattr(self,key,value):
-        _allowed = ['host','port','db','doc','authSource']
+        _allowed = ['host','port','db','doc','authSource','mechanism']
         if key in _allowed :
             setattr(self,key,value)
         pass
@@ -92,10 +92,10 @@ class MongoReader(Mongo,Reader):
         Mongo.__init__(self,**args)
     def read(self,**args):
         
-        if 'mongo' in args :
+        if 'mongo' in args or 'cmd' :
             #
             # @TODO:
-            cmd = args['mongo']
+            cmd = args['mongo'] if 'mongo' in args else args['cmd']
             if "aggregate" in cmd :
                 if "allowDiskUse" not in cmd :
                     cmd["allowDiskUse"] = True

+ 3 - 0
transport/sql.py

@@ -430,6 +430,9 @@ class BQWriter(BigQuery,Writer):
                 self.mode['destination_table'] = _args['table'].strip()
             if 'schema' in _args :
                 self.mode['table_schema'] = _args['schema']
+                #
+                # Let us insure that the types are somewhat compatible ...
+                # _map = {'INTEGER':np.int64,'DATETIME':'datetime64[ns]','TIMESTAMP':'datetime64[ns]','FLOAT':np.float64,'DOUBLE':np.float64,'STRING':str}
             # _mode = copy.deepcopy(self.mode)
             _mode = self.mode
             # _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)