common.py 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. """
  2. Data Transport - 1.0
  3. Steve L. Nyemba, The Phi Technology LLC
  4. This module is designed to serve as a wrapper to a set of supported data stores :
  5. - couchdb
  6. - mongodb
  7. - Files (character delimited)
  8. - Queues (Rabbmitmq)
  9. - Session (Flask)
  10. - s3
  11. The supported operations are read/write and providing meta data to the calling code
  12. Requirements :
  13. pymongo
  14. boto
  15. couldant
  16. @TODO:
  17. Enable read/writing to multiple reads/writes
  18. """
  19. __author__ = 'The Phi Technology'
  20. import numpy as np
  21. import json
  22. import importlib
  23. from multiprocessing import RLock
  24. import queue
  25. # import couch
  26. # import mongo
  27. class IO:
  28. def init(self,**args):
  29. """
  30. This function enables attributes to be changed at runtime. Only the attributes defined in the class can be changed
  31. Adding attributes will require sub-classing otherwise we may have an unpredictable class ...
  32. """
  33. allowed = list(vars(self).keys())
  34. for field in args :
  35. if field not in allowed :
  36. continue
  37. value = args[field]
  38. setattr(self,field,value)
  39. class Reader (IO):
  40. """
  41. This class is an abstraction of a read functionalities of a data store
  42. """
  43. def __init__(self):
  44. pass
  45. def meta(self,**_args):
  46. """
  47. This function is intended to return meta-data associated with what has just been read
  48. @return object of meta data information associated with the content of the store
  49. """
  50. raise Exception ("meta function needs to be implemented")
  51. def read(self,**args):
  52. """
  53. This function is intended to read the content of a store provided parameters to be used at the discretion of the subclass
  54. """
  55. raise Exception ("read function needs to be implemented")
  56. class Writer(IO):
  57. def __init__(self):
  58. self.cache = {"default":[]}
  59. def log(self,**args):
  60. self.cache[id] = args
  61. def meta (self,id="default",**args):
  62. raise Exception ("meta function needs to be implemented")
  63. def format(self,row,xchar):
  64. if xchar is not None and isinstance(row,list):
  65. return xchar.join(row)+'\n'
  66. elif xchar is None and isinstance(row,dict):
  67. row = json.dumps(row)
  68. return row
  69. def write(self,**args):
  70. """
  71. This function will write content to a store given parameters to be used at the discretion of the sub-class
  72. """
  73. raise Exception ("write function needs to be implemented")
  74. def archive(self):
  75. """
  76. It is important to be able to archive data so as to insure that growth is controlled
  77. Nothing in nature grows indefinitely neither should data being handled.
  78. """
  79. raise Exception ("archive function needs to be implemented")
  80. def close(self):
  81. """
  82. This function will close the persistent storage connection/handler
  83. """
  84. pass
  85. class ReadWriter(Reader,Writer) :
  86. """
  87. This class implements the read/write functions aggregated
  88. """
  89. pass
  90. class Console(Writer):
  91. lock = RLock()
  92. def __init__(self,**_args):
  93. self.lock = _args['lock'] if 'lock' in _args else False
  94. self.info = self.write
  95. self.debug = self.write
  96. self.log = self.write
  97. pass
  98. def write (self,logs=None,**_args):
  99. if self.lock :
  100. Console.lock.acquire()
  101. try:
  102. _params = _args if logs is None and _args else logs
  103. if type(_params) == list:
  104. for row in _params :
  105. print (row)
  106. else:
  107. print (_params)
  108. except Exception as e :
  109. print (e)
  110. finally:
  111. if self.lock :
  112. Console.lock.release()
  113. """
  114. @NOTE : Experimental !!
  115. """
  116. class Proxy :
  117. """
  118. This class will forward a call to a function that is provided by the user code
  119. """
  120. def __init__(self,**_args):
  121. self.callback = _args['callback']
  122. def read(self,**_args) :
  123. try:
  124. return self.callback(**_args)
  125. except Exception as e:
  126. return self.callback()
  127. pass
  128. def write(self,data,**_args):
  129. self.callback(data,**_args)