__init__.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. """
  2. (c) 2019 Healthcare/IO 1.0
  3. Vanderbilt University Medical Center, Health Information Privacy Laboratory
  4. https://hiplab.mc.vanderbilt.edu/healthcareio
  5. Authors:
  6. Khanhly Nguyen,
  7. Steve L. Nyemba<steve.l.nyemba@vanderbilt.edu>
  8. License:
  9. MIT, terms are available at https://opensource.org/licenses/MIT
  10. This parser was originally written by Khanhly Nguyen for her internship and is intended to parse x12 835,837 and others provided the appropriate configuration
  11. USAGE :
  12. - COMMAND LINE
  13. - EMBEDDED
  14. """
  15. import hashlib
  16. import json
  17. import os
  18. import sys
  19. from itertools import islice
  20. from multiprocessing import Process
  21. import transport
  22. import jsonmerge
  23. class void :
  24. pass
  25. class Formatters :
  26. def __init__(self):
  27. # self.config = config
  28. self.get = void()
  29. self.get.config = self.get_config
  30. self.parse = void()
  31. self.parse.sv3 = self.sv3
  32. self.parse.sv2 = self.sv2
  33. self.sv2_parser = self.sv2
  34. self.sv3_parser = self.sv3
  35. self.sv3_parse = self.sv3
  36. self.format_proc = self.procedure
  37. self.format_diag = self.diagnosis
  38. self.parse.procedure = self.procedure
  39. self.parse.diagnosis = self.diagnosis
  40. self.parse.date = self.date
  41. self.format_date = self.date
  42. self.format_pos = self.pos
  43. self.format_time = self.time
  44. def split(self,row,sep='*',prefix='HI') :
  45. """
  46. This function is designed to split an x12 row and
  47. """
  48. value = []
  49. if row.startswith(prefix) is False:
  50. for row_value in row.replace('~','').split(sep) :
  51. if '>' in row_value and not row_value.startswith('HC'):
  52. # if row_value.startswith('HC') or row_value.startswith('AD'):
  53. if row_value.startswith('AD'):
  54. value += row_value.split('>')[:2]
  55. pass
  56. else:
  57. value += [row_value]
  58. # value += row_value.split('>') if row.startswith('CLM') is False else [row_value]
  59. else :
  60. value.append(row_value.replace('\n',''))
  61. value = [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep)
  62. else:
  63. value = [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ]
  64. return value if type(value) == list and type(value[0]) != list else value[0]
  65. def get_config(self,config,row):
  66. """
  67. This function will return the meaningfull parts of the configuration for a given item
  68. """
  69. _row = list(row) if type(row[0]) == str else list(row[0])
  70. _info = config[_row[0]] if _row[0] in config else {}
  71. key = None
  72. if '@ref' in _info:
  73. key = list(set(_row) & set(_info['@ref'].keys()))
  74. if key :
  75. key = key[0]
  76. return _info['@ref'][key]
  77. else:
  78. return {}
  79. if not _info and 'SIMILAR' in config:
  80. #
  81. # Let's look for the nearest key using the edit distance
  82. if _row[0] in config['SIMILAR'] :
  83. key = config['SIMILAR'][_row[0]]
  84. _info = config[key]
  85. return _info
  86. def hash(self,value):
  87. salt = os.environ['HEALTHCAREIO_SALT'] if 'HEALTHCAREIO_SALT' in os.environ else ''
  88. _value = str(value)+ salt
  89. if sys.version_info[0] > 2 :
  90. return hashlib.md5(_value.encode('utf-8')).hexdigest()
  91. else:
  92. return hashlib.md5(_value).hexdigest()
  93. def suppress (self,value):
  94. return 'N/A'
  95. def date(self,value):
  96. if len(value) > 8 or '-' in value:
  97. value = value.split('-')[0]
  98. if len(value) == 8 :
  99. year = value[:4]
  100. month = value[4:6]
  101. day = value[6:]
  102. return "-".join([year,month,day])[:10] #{"year":year,"month":month,"day":day}
  103. elif len(value) == 6 :
  104. year = '20' + value[:2]
  105. month = value[2:4]
  106. day = value[4:]
  107. #
  108. # We have a date formatting issue
  109. return "-".join([year,month,day])
  110. def time(self,value):
  111. pass
  112. def sv3(self,value):
  113. if '>' in value [1]:
  114. terms = value[1].split('>')
  115. return {'type':terms[0],'code':terms[1],"amount":float(value[2])}
  116. else:
  117. return {"code":value[2],"type":value[1],"amount":float(value[3])}
  118. def sv2(self,value):
  119. #
  120. # @TODO: Sometimes there's a suffix (need to inventory all the variations)
  121. #
  122. if '>' in value or ':' in value:
  123. xchar = '>' if '>' in value else ':'
  124. _values = value.split(xchar)
  125. modifier = {}
  126. if len(_values) > 2 :
  127. modifier= {"code":_values[2]}
  128. if len(_values) > 3 :
  129. modifier['type'] = _values[3]
  130. _value = {"code":_values[1],"type":_values[0]}
  131. if modifier :
  132. _value['modifier'] = modifier
  133. return _value
  134. else:
  135. return value
  136. def procedure(self,value):
  137. for xchar in [':','<','|','>'] :
  138. if xchar in value and len(value.split(xchar)) > 1 :
  139. #_value = {"type":value.split(':')[0].strip(),"code":value.split(':')[1].strip()}
  140. _value = {"type":value.split(xchar)[0].strip(),"code":value.split(xchar)[1].strip()}
  141. if len(value.split(xchar)) >2 :
  142. index = 1;
  143. for modifier in value.split(xchar)[2:] :
  144. _value['modifier_'+str(index)] = modifier
  145. index += 1
  146. break
  147. else:
  148. _value = str(value)
  149. return _value
  150. def diagnosis(self,value):
  151. return [ {"code":item[2], "type":item[1]} for item in value if len(item) > 1]
  152. def pos(self,value):
  153. """
  154. formatting place of service information within a segment (REF)
  155. @TODO: In order to accomodate the other elements they need to be specified in the configuration
  156. Otherwise it causes problems on export
  157. """
  158. xchar = '>' if '>' in value else ':'
  159. x = value.split(xchar)
  160. x = {"code":x[0],"indicator":x[1],"frequency":x[2]} if len(x) == 3 else {"code":x[0],"indicator":None,"frequency":None}
  161. return x['code']
  162. class Parser (Process):
  163. def __init__(self,path):
  164. """
  165. :path path of the configuration file (it can be absolute)
  166. """
  167. Process.__init__(self)
  168. self.utils = Formatters()
  169. self.get = void()
  170. self.get.value = self.get_map
  171. self.get.default_value = self.get_default_value
  172. _config = json.loads(open(path).read())
  173. self._custom_config = self.get_custom(path)
  174. self.config = _config['parser']
  175. self.store = _config['store']
  176. self.files = []
  177. self.set = void()
  178. self.set.files = self.set_files
  179. self.emit = void()
  180. self.emit.pre = None
  181. self.emit.post = None
  182. def get_custom(self,path) :
  183. """
  184. :path path of the configuration file (it can be absolute)
  185. """
  186. #
  187. #
  188. _path = path.replace('config.json','')
  189. if _path.endswith(os.sep) :
  190. _path = _path[:-1]
  191. _config = {}
  192. _path = os.sep.join([_path,'custom'])
  193. if os.path.exists(_path) :
  194. files = os.listdir(_path)
  195. if files :
  196. fullname = os.sep.join([_path,files[0]])
  197. _config = json.loads ( (open(fullname)).read() )
  198. return _config
  199. def set_files(self,files):
  200. self.files = files
  201. def get_map(self,row,config,version=None):
  202. # label = config['label'] if 'label' in config else None
  203. handler = Formatters()
  204. if 'map' not in config and hasattr(handler,config['apply']):
  205. pointer = getattr(handler,config['apply'])
  206. object_value = pointer(row)
  207. return object_value
  208. #
  209. # Pull the goto configuration that skips rows
  210. #
  211. omap = config['map'] if not version or version not in config else config[version]
  212. anchors = config['anchors'] if 'anchors' in config else []
  213. rewrite = config['rewrite'] if 'rewrite' in config else {}
  214. if type(row[0]) == str:
  215. object_value = {}
  216. for key in omap :
  217. index = omap[key]
  218. if anchors and set(anchors) & set(row):
  219. _key = list(set(anchors) & set(row))[0]
  220. aindex = row.index(_key)
  221. index = aindex + index
  222. if index < len(row) :
  223. value = row[index]
  224. if 'cast' in config and key in config['cast'] and value.strip() != '' :
  225. if config['cast'][key] in ['float','int'] :
  226. value = eval(config['cast'][key])(value)
  227. elif hasattr(handler,config['cast'][key]):
  228. pointer = getattr(handler,config['cast'][key])
  229. value = pointer(value)
  230. else:
  231. print ("Missing Pointer ",key,config['cast'])
  232. if type(value) == dict :
  233. for objkey in value :
  234. if type(value[objkey]) == dict :
  235. continue
  236. if 'syn' in config and value[objkey] in config['syn'] :
  237. value[objkey] = config['syn'][ value[objkey]]
  238. if key in rewrite :
  239. _key = rewrite[key]
  240. if _key in value :
  241. value = value[_key]
  242. else:
  243. value = ""
  244. value = {key:value} if key not in value else value
  245. else:
  246. if 'syn' in config and value in config['syn'] :
  247. value = config['syn'][value]
  248. if type(value) == dict :
  249. object_value = dict(object_value, **value)
  250. else:
  251. object_value[key] = value
  252. else:
  253. #
  254. # we are dealing with a complex object
  255. object_value = []
  256. for row_item in row :
  257. value = self.get.value(row_item,config,version)
  258. object_value.append(value)
  259. #
  260. # We need to add the index of the object it matters in determining the claim types
  261. #
  262. # object_value.append( list(get_map(row_item,config,version)))
  263. # object_value = {label:object_value}
  264. return object_value
  265. def apply(self,content,_code) :
  266. """
  267. :content content of a file i.e a segment with the envelope
  268. :_code 837 or 835 (helps get the appropriate configuration)
  269. """
  270. util = Formatters()
  271. # header = default_value.copy()
  272. value = {}
  273. for row in content[:] :
  274. row = util.split(row.replace('\n','').replace('~',''))
  275. _info = util.get.config(self.config[_code][0],row)
  276. if self._custom_config and _code in self._custom_config:
  277. _cinfo = util.get.config(self._custom_config[_code],row)
  278. else:
  279. _cinfo = {}
  280. if 'SV3' in row :
  281. print (row)
  282. print (_info)
  283. if _info or _cinfo:
  284. try:
  285. _info = jsonmerge.merge(_info,_cinfo)
  286. tmp = self.get.value(row,_info)
  287. if not tmp :
  288. continue
  289. if 'label' in _info :
  290. label = _info['label']
  291. if type(tmp) == list :
  292. value[label] = tmp if label not in value else value[label] + tmp
  293. else:
  294. if label not in value:
  295. value[label] = [tmp]
  296. else:
  297. value[label].append(tmp)
  298. tmp['_index'] = len(value[label]) -1
  299. elif 'field' in _info :
  300. name = _info['field']
  301. # value[name] = tmp
  302. value = jsonmerge.merge(value,{name:tmp})
  303. else:
  304. value = dict(value,**tmp)
  305. pass
  306. except Exception as e :
  307. print (e.args[0])
  308. # print ('__',(dir(e.args)))
  309. pass
  310. return value if value else {}
  311. def get_default_value(self,content,_code):
  312. util = Formatters()
  313. TOP_ROW = content[1].split('*')
  314. CATEGORY= content[2].split('*')[1].strip()
  315. VERSION = content[1].split('*')[-1].replace('~','').replace('\n','')
  316. SUBMITTED_DATE = util.parse.date(TOP_ROW[4])
  317. SENDER_ID = TOP_ROW[2]
  318. row = util.split(content[3])
  319. _info = util.get_config(self.config[_code][0],row)
  320. value = self.get.value(row,_info,VERSION) if _info else {}
  321. value['category'] = {"setid": CATEGORY,"version":'X'+VERSION.split('X')[1],"id":VERSION.split('X')[0].strip()}
  322. value["submitted"] = SUBMITTED_DATE
  323. # value['version'] = VERSION
  324. # if _code== '835' :
  325. # value['receiver_id'] = SENDER_ID
  326. # else:
  327. # value['provider_id'] = SENDER_ID
  328. # pass
  329. value['sender_id'] = SENDER_ID
  330. #
  331. # Let's parse this for default values
  332. return value
  333. def read(self,filename) :
  334. """
  335. :formerly get_content
  336. This function returns the of the EDI file parsed given the configuration specified. it is capable of identifying a file given the content
  337. :section loop prefix (HL, CLP)
  338. :config configuration with formatting rules, labels ...
  339. :filename location of the file
  340. """
  341. # section = section if section else config['SECTION']
  342. logs = []
  343. claims = []
  344. try:
  345. file = open(filename.strip(),errors='ignore')
  346. INITIAL_ROWS = list(islice(file,4)) #.readlines(4)
  347. _code = "unknown"
  348. if len(INITIAL_ROWS) == 1 :
  349. file = INITIAL_ROWS[0].split('~')
  350. INITIAL_ROWS = file[:4]
  351. if len(INITIAL_ROWS) < 3 :
  352. return None,[{"name":filename,"completed":False}],None
  353. _code = INITIAL_ROWS[2].split('*')[1].strip()
  354. section = self.config[_code][0]['SECTION'].strip()
  355. #
  356. # adjusting the
  357. DEFAULT_VALUE = self.get.default_value(INITIAL_ROWS,_code)
  358. DEFAULT_VALUE['name'] = filename.strip()
  359. #
  360. # In the initial rows, there's redundant information (so much for x12 standard)
  361. # index 1 identifies file type i.e CLM for claim and CLP for remittance
  362. segment = []
  363. index = 0;
  364. _toprows = []
  365. for row in file :
  366. row = row.replace('\r','')
  367. if not segment and not row.startswith(section):
  368. _toprows += [row]
  369. if row.startswith(section) and not segment:
  370. segment = [row]
  371. continue
  372. elif segment and not row.startswith(section):
  373. segment.append(row)
  374. if len(segment) > 1 and row.startswith(section):
  375. #
  376. # process the segment somewhere (create a thread maybe?)
  377. #
  378. _claim = self.apply(segment,_code)
  379. if _claim :
  380. _claim['index'] = index #len(claims)
  381. claims.append(dict(DEFAULT_VALUE,**_claim))
  382. segment = [row]
  383. index += 1
  384. pass
  385. #
  386. # Handling the last claim found
  387. if segment[0].startswith(section) :
  388. default_claim = dict({"name":index},**DEFAULT_VALUE)
  389. claim = self.apply(segment,_code)
  390. if claim :
  391. claim['index'] = len(claims)
  392. # for _row_ in _toprows :
  393. # claim = jsonmerge.merge(claim,self.apply([_row_],_code))
  394. claim = jsonmerge.merge(claim,self.apply(_toprows,_code))
  395. claims.append(dict(DEFAULT_VALUE,**claim))
  396. if type(file) != list :
  397. file.close()
  398. # x12_file = open(filename.strip(),errors='ignore').read().split('\n')
  399. except Exception as e:
  400. logs.append ({"parse":_code,"completed":False,"name":filename,"msg":e.args[0]})
  401. return [],logs,None
  402. rate = 0 if len(claims) == 0 else (1 + index)/len(claims)
  403. logs.append ({"parse":"claims" if _code == '837' else 'remits',"completed":True,"name":filename,"rate":rate})
  404. # self.finish(claims,logs,_code)
  405. return claims,logs,_code
  406. def run(self):
  407. if self.emit.pre :
  408. self.emit.pre()
  409. for filename in self.files :
  410. content,logs,_code = self.read(filename)
  411. self.finish(content,logs,_code)
  412. def finish(self,content,logs,_code) :
  413. args = self.store
  414. _args = json.loads(json.dumps(self.store))
  415. if args['type'] == 'mongo.MongoWriter' :
  416. args['args']['doc'] = 'claims' if _code == '837' else 'remits'
  417. _args['args']['doc'] = 'logs'
  418. else:
  419. args['args']['table'] = 'claims' if _code == '837' else 'remits'
  420. _args['args']['table'] = 'logs'
  421. if content :
  422. writer = transport.factory.instance(**args)
  423. writer.write(content)
  424. writer.close()
  425. if logs :
  426. logger = transport.factory.instance(**_args)
  427. logger.write(logs)
  428. logger.close()
  429. if self.emit.post :
  430. self.emit.post(content,logs)