Merge pull request #40 from lnyemba/v2.2.0

V2.2.0

transport/cloud/s3.py

@@ -12,7 +12,7 @@ import boto3
 # from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 import numpy as np
 import botocore
-from smart_open import smart_open
+# from smart_open import smart_open
 import sys
 
 import json

transport/info.py

@@ -1,6 +1,6 @@
 __app_name__  = 'data-transport'
 __author__ = 'The Phi Technology'
-__version__= '2.2.46'
+__version__= '2.2.50'
 __email__  = "info@the-phi.com"
 __edition__= 'community'
 __license__=f"""

transport/other/files.py

@@ -20,6 +20,16 @@ class File :
 	def isready(self):
 		return os.path.exists(self.path) 
 	def meta(self,**_args):
+		"""
+		This function will return the schema for a given file
+		"""
+		_path = self.path if 'path' not in _args else _args['path']
+		if os.path.exists(_path):
+			_hdf = pd.read_csv(_path,sep=self.delimiter,nrows=10)
+			_names = _hdf.dtypes.index.tolist()[1:]
+			_types = [_x.name for _x in _hdf.dtypes.tolist()[1:]]
+			return pd.DataFrame({"name":_names,"type":_types}).to_dict(orient='records')
+
 		return []
 	
 class Reader (File):
@@ -35,8 +45,12 @@ class Reader (File):
 		_path = self.path if 'path' not in args else args['path']
 		_delimiter = self.delimiter if 'delimiter' not in args else args['delimiter']
 		_df =  pd.read_csv(_path,delimiter=_delimiter)
-		if 'query' in args :
+		_query = None
+		if 'sql' in args :
+			_query = args['sql']
+		elif 'query' in args :
 			_query = args['query']
+		if _query :
 			_df = _df.query(_query)
 		return _df
 	def stream(self,**args):