|
|
@@ -11,6 +11,9 @@ from couchdbkit import Server
|
|
|
import re
|
|
|
from csv import reader
|
|
|
from datetime import datetime
|
|
|
+import boto
|
|
|
+import botocore
|
|
|
+from smart_open import smart_open
|
|
|
"""
|
|
|
@TODO: Write a process by which the class automatically handles reading and creating a preliminary sample and discovers the meta data
|
|
|
"""
|
|
|
@@ -629,12 +632,81 @@ class CouchdbWriter(Couchdb,Writer):
|
|
|
name = '-'.join([document['_id'] , now,'.json'])
|
|
|
self.dbase.save_doc(document)
|
|
|
self.dbase.put_attachment(document,content,name,'application/json')
|
|
|
+class s3 :
|
|
|
+ """
|
|
|
+ @TODO: Implement a search function for a file given a bucket??
|
|
|
+ """
|
|
|
+ def __init__(self,args) :
|
|
|
+ """
|
|
|
+ This function will extract a file or set of files from s3 bucket provided
|
|
|
+ @param access_key
|
|
|
+ @param secret_key
|
|
|
+ @param path location of the file
|
|
|
+ @param filter filename or filtering elements
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ self.s3 = boto.connect_s3(args['access_key'],args['secret_key'])
|
|
|
+ self.bucket = self.s3.get_bucket(args['bucket'].strip(),validate=False) if 'bucket' in args else None
|
|
|
+ # self.path = args['path']
|
|
|
+ self.filter = args['filter'] if 'filter' in args else None
|
|
|
+ self.filename = args['file'] if 'file' in args else None
|
|
|
+
|
|
|
+ except Exception as e :
|
|
|
+ self.s3 = None
|
|
|
+ self.bucket = None
|
|
|
+ print e
|
|
|
+ def buckets(self):
|
|
|
+ """
|
|
|
+ This function is a wrapper around the bucket list of buckets for s3
|
|
|
+
|
|
|
+ """
|
|
|
+ return self.s3.get_all_buckets()
|
|
|
+
|
|
|
+
|
|
|
+class s3Reader(s3,Reader) :
|
|
|
+ """
|
|
|
+ Because s3 contains buckets and files, reading becomes a tricky proposition :
|
|
|
+ - list files if file is None
|
|
|
+ - stream content if file is Not None
|
|
|
+ @TODO: support read from all buckets, think about it
|
|
|
+ """
|
|
|
+ def __init__(self,args) :
|
|
|
+ s3.__init__(self,args)
|
|
|
+ def files(self):
|
|
|
+ r = []
|
|
|
+ try:
|
|
|
+ return [item.name for item in self.bucket if item.size > 0]
|
|
|
+ except Exception as e:
|
|
|
+ pass
|
|
|
+ return r
|
|
|
+ def stream(self,limit=-1):
|
|
|
+ """
|
|
|
+ At this point we should stream a file from a given bucket
|
|
|
+ """
|
|
|
+ key = self.bucket.get_key(self.filename.strip())
|
|
|
+ if key is None :
|
|
|
+ yield None
|
|
|
+ else:
|
|
|
+ count = 0
|
|
|
+ with smart_open(key) as remote_file:
|
|
|
+ for line in remote_file:
|
|
|
+ if count == limit and limit > 0 :
|
|
|
+ break
|
|
|
+ yield line
|
|
|
+ count += 1
|
|
|
+ def read(self,limit=-1) :
|
|
|
+ if self.filename is None :
|
|
|
+ #
|
|
|
+ # returning the list of files because no one file was specified.
|
|
|
+ return self.files()
|
|
|
+ else:
|
|
|
+ return self.stream(10)
|
|
|
"""
|
|
|
This class acts as a factory to be able to generate an instance of a Reader/Writer
|
|
|
Against a Queue,Disk,Cloud,Couchdb
|
|
|
The class doesn't enforce parameter validation, thus any error with the parameters sent will result in a null Object
|
|
|
"""
|
|
|
-class DataSourceFactory:
|
|
|
+class Factory:
|
|
|
def instance(self,**args):
|
|
|
source = args['type']
|
|
|
params = args['args']
|
|
|
@@ -659,6 +731,10 @@ class DataSourceFactory:
|
|
|
except Exception,e:
|
|
|
print ['Error ',e]
|
|
|
return anObject
|
|
|
+class s3Writer(s3,Writer) :
|
|
|
+ def __init__(self,args) :
|
|
|
+ s3.__init__(self,args)
|
|
|
+
|
|
|
"""
|
|
|
This class implements a data-source handler that is intended to be used within the context of data processing, it allows to read/write anywhere transparently.
|
|
|
The class is a facade to a heterogeneous class hierarchy and thus simplifies how the calling code interacts with the class hierarchy
|
|
|
@@ -671,16 +747,12 @@ class DataSource:
|
|
|
return self.Input.read(size)
|
|
|
def write(self,**args):
|
|
|
self.Output.write(**args)
|
|
|
-#p = {}
|
|
|
-#p['host'] = 'dev.the-phi.com'
|
|
|
-#p['uid'] = 'nyemba@gmail.com'
|
|
|
-#p['qid'] = 'repair'
|
|
|
-#factory = DataSourceFactory()
|
|
|
-#o = factory.instance(type='QueueReader',args=p)
|
|
|
-#print o is None
|
|
|
-#q = QueueWriter(host='dev.the-phi.com',uid='nyemba@gmail.com')
|
|
|
-#q.write(object='steve')
|
|
|
-#q.write(object='nyemba')
|
|
|
-#q.write(object='elon')
|
|
|
-
|
|
|
-
|
|
|
+conf = json.loads(open('config.json').read())
|
|
|
+#x = s3Reader( dict(conf,**{'bucket':'com.phi.sample.data','file':'Sample-Spreadsheet-5000-rows.csv'}))
|
|
|
+x = s3Reader(conf)
|
|
|
+print conf
|
|
|
+print x.bucket.get_all_keys()
|
|
|
+# r = x.read()
|
|
|
+# for item in r :
|
|
|
+# print item
|
|
|
+#print buckets[1].get_key('Sample-Spreadsheet-5000-rows.csv')
|
