|
@@ -2,15 +2,29 @@
|
|
|
"cells": [
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
- "execution_count": 66,
|
|
|
+ "execution_count": 1,
|
|
|
"metadata": {},
|
|
|
- "outputs": [],
|
|
|
+ "outputs": [
|
|
|
+ {
|
|
|
+ "name": "stdout",
|
|
|
+ "output_type": "stream",
|
|
|
+ "text": [
|
|
|
+ "dev-deid-600@aou-res-deid-vumc-test.iam.gserviceaccount.com df0ac049-d5b6-416f-ab3c-6321eda919d6 2018-09-25 08:18:34.829000+00:00 DONE\n"
|
|
|
+ ]
|
|
|
+ }
|
|
|
+ ],
|
|
|
"source": [
|
|
|
"import pandas as pd\n",
|
|
|
"import numpy as np\n",
|
|
|
"from google.cloud import bigquery as bq\n",
|
|
|
"\n",
|
|
|
- "client = bq.Client.from_service_account_json('/home/steve/dev/google-cloud-sdk/accounts/vumc-test.json')"
|
|
|
+ "client = bq.Client.from_service_account_json('/home/steve/dev/google-cloud-sdk/accounts/vumc-test.json')\n",
|
|
|
+ "# pd.read_gbq(query=\"select * from raw.observation limit 10\",private_key='/home/steve/dev/google-cloud-sdk/accounts/vumc-test.json')\n",
|
|
|
+ "jobs = client.list_jobs()\n",
|
|
|
+ "for job in jobs :\n",
|
|
|
+ "# print dir(job)\n",
|
|
|
+ " print job.user_email,job.job_id,job.started, job.state\n",
|
|
|
+ " break"
|
|
|
]
|
|
|
},
|
|
|
{
|
|
@@ -25,7 +39,7 @@
|
|
|
},
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
- "execution_count": 181,
|
|
|
+ "execution_count": 10,
|
|
|
"metadata": {},
|
|
|
"outputs": [],
|
|
|
"source": [
|
|
@@ -68,7 +82,7 @@
|
|
|
" else:\n",
|
|
|
" x_ = args['xi']\n",
|
|
|
" for xi in x_ :\n",
|
|
|
- " fields += (['.'.join([xi['name'],name]) for name in xi['fields'] if name != args['join']])\n",
|
|
|
+ " fields += (['.'.join([xi['name'], name]) for name in xi['fields'] if name != args['join']])\n",
|
|
|
" return fields\n",
|
|
|
"def generate_sql(**args):\n",
|
|
|
" \"\"\"\n",
|
|
@@ -97,7 +111,27 @@
|
|
|
" tmp.append(ON_SQL)\n",
|
|
|
" INNER_JOINS += [JOIN_SQL + \" AND \".join(tmp)]\n",
|
|
|
" return SQL + \" \".join(INNER_JOINS)\n",
|
|
|
- " \n",
|
|
|
+ "def get_final_sql(**args):\n",
|
|
|
+ " xo = args['xo']\n",
|
|
|
+ " xi = args['xi']\n",
|
|
|
+ " join=args['join']\n",
|
|
|
+ " prefix = args['prefix'] if 'prefix' in args else ''\n",
|
|
|
+ " fields = get_fields (xo=xo,xi=xi,join=join)\n",
|
|
|
+ " k = len(fields)\n",
|
|
|
+ " n = np.random.randint(2,k) #-- number of fields to select\n",
|
|
|
+ " i = np.random.randint(0,k,size=n)\n",
|
|
|
+ " fields = [name for name in fields if fields.index(name) in i]\n",
|
|
|
+ " base_sql = generate_sql(xo=xo,xi=xi,prefix)\n",
|
|
|
+ " SQL = \"\"\"\n",
|
|
|
+ " SELECT AVERAGE(count),size,n as selected_features,k as total_features\n",
|
|
|
+ " FROM(\n",
|
|
|
+ " SELECT COUNT(*) as count,count(:join) as pop,sum(:n) as N,sum(:k) as k,:fields\n",
|
|
|
+ " FROM (:sql)\n",
|
|
|
+ " GROUP BY :fields\n",
|
|
|
+ " ) \n",
|
|
|
+ " order by 1\n",
|
|
|
+ " \n",
|
|
|
+ " \"\"\".replace(\":sql\",base_sql)\n",
|
|
|
"# sql = \"SELECT :fields FROM :xo.name INNER JOIN :xi.name ON :xi.name.:xi.y = :xo.y \"\n",
|
|
|
"# fields = \",\".join(get_fields(xo=xi,xi=xi,join=xi['y']))\n",
|
|
|
" \n",
|
|
@@ -111,24 +145,39 @@
|
|
|
},
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
- "execution_count": 183,
|
|
|
+ "execution_count": 33,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "xo = {\"name\":\"person\",\"fields\":['person_id','date_of_birth','race','value_as_number']}\n",
|
|
|
+ "xi = [{\"name\":\"measurement\",\"fields\":['person_id','value_as_number','value_source_value']}] #,{\"name\":\"observation\",\"fields\":[\"person_id\",\"value_as_string\",\"observation_source_value\"]}]\n",
|
|
|
+ "# generate_sql(xo=xo,xi=xi,join=\"person_id\",prefix='raw')\n",
|
|
|
+ "fields = get_fields(xo=xo,xi=xi,join='person_id')\n",
|
|
|
+ "ofields = list(fields)\n",
|
|
|
+ "k = len(fields)\n",
|
|
|
+ "n = np.random.randint(2,k) #-- number of fields to select\n",
|
|
|
+ "i = np.random.randint(0,k,size=n)\n",
|
|
|
+ "fields = [name for name in fields if fields.index(name) in i]"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 34,
|
|
|
"metadata": {},
|
|
|
"outputs": [
|
|
|
{
|
|
|
"data": {
|
|
|
"text/plain": [
|
|
|
- "'SELECT :fields FROM raw.person INNER JOIN raw.measurement ON measurement.person_id = person.person_id'"
|
|
|
+ "['person.race', 'person.value_as_number', 'measurement.value_source_value']"
|
|
|
]
|
|
|
},
|
|
|
- "execution_count": 183,
|
|
|
+ "execution_count": 34,
|
|
|
"metadata": {},
|
|
|
"output_type": "execute_result"
|
|
|
}
|
|
|
],
|
|
|
"source": [
|
|
|
- "xo = {\"name\":\"person\",\"fields\":['person_id','date_of_birth','race']}\n",
|
|
|
- "xi = [{\"name\":\"measurement\",\"fields\":['person_id','value_as_number','value_source_value']}] #,{\"name\":\"observation\",\"fields\":[\"person_id\",\"value_as_string\",\"observation_source_value\"]}]\n",
|
|
|
- "generate_sql(xo=xo,xi=xi,join=\"person_id\",prefix='raw')"
|
|
|
+ "fields\n"
|
|
|
]
|
|
|
},
|
|
|
{
|
|
@@ -179,69 +228,16 @@
|
|
|
},
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
- "execution_count": 111,
|
|
|
+ "execution_count": 6,
|
|
|
"metadata": {},
|
|
|
"outputs": [
|
|
|
{
|
|
|
"data": {
|
|
|
"text/plain": [
|
|
|
- "[u'condition_occurrence.condition_occurrence_id',\n",
|
|
|
- " u'condition_occurrence.person_id',\n",
|
|
|
- " u'condition_occurrence.condition_concept_id',\n",
|
|
|
- " u'condition_occurrence.condition_start_date',\n",
|
|
|
- " u'condition_occurrence.condition_start_datetime',\n",
|
|
|
- " u'condition_occurrence.condition_end_date',\n",
|
|
|
- " u'condition_occurrence.condition_end_datetime',\n",
|
|
|
- " u'condition_occurrence.condition_type_concept_id',\n",
|
|
|
- " u'condition_occurrence.stop_reason',\n",
|
|
|
- " u'condition_occurrence.provider_id',\n",
|
|
|
- " u'condition_occurrence.visit_occurrence_id',\n",
|
|
|
- " u'condition_occurrence.condition_source_value',\n",
|
|
|
- " u'condition_occurrence.condition_source_concept_id',\n",
|
|
|
- " u'death.death_date',\n",
|
|
|
- " u'death.death_datetime',\n",
|
|
|
- " u'death.death_type_concept_id',\n",
|
|
|
- " u'death.cause_concept_id',\n",
|
|
|
- " u'death.cause_source_value',\n",
|
|
|
- " u'death.cause_source_concept_id',\n",
|
|
|
- " u'device_exposure.device_exposure_id',\n",
|
|
|
- " u'device_exposure.device_concept_id',\n",
|
|
|
- " u'device_exposure.device_exposure_start_date',\n",
|
|
|
- " u'device_exposure.device_exposure_start_datetime',\n",
|
|
|
- " u'device_exposure.device_exposure_end_date',\n",
|
|
|
- " u'device_exposure.device_exposure_end_datetime',\n",
|
|
|
- " u'device_exposure.device_type_concept_id',\n",
|
|
|
- " u'device_exposure.unique_device_id',\n",
|
|
|
- " u'device_exposure.quantity',\n",
|
|
|
- " u'device_exposure.provider_id',\n",
|
|
|
- " u'device_exposure.visit_occurrence_id',\n",
|
|
|
- " u'device_exposure.device_source_value',\n",
|
|
|
- " u'device_exposure.device_source_concept_id',\n",
|
|
|
- " u'drug_exposure.drug_exposure_id',\n",
|
|
|
- " u'drug_exposure.drug_concept_id',\n",
|
|
|
- " u'drug_exposure.drug_exposure_start_date',\n",
|
|
|
- " u'drug_exposure.drug_exposure_start_datetime',\n",
|
|
|
- " u'drug_exposure.drug_exposure_end_date',\n",
|
|
|
- " u'drug_exposure.drug_exposure_end_datetime',\n",
|
|
|
- " u'drug_exposure.drug_type_concept_id',\n",
|
|
|
- " u'drug_exposure.stop_reason',\n",
|
|
|
- " u'drug_exposure.refills',\n",
|
|
|
- " u'drug_exposure.quantity',\n",
|
|
|
- " u'drug_exposure.days_supply',\n",
|
|
|
- " u'drug_exposure.sig',\n",
|
|
|
- " u'drug_exposure.route_concept_id',\n",
|
|
|
- " u'drug_exposure.effective_drug_dose',\n",
|
|
|
- " u'drug_exposure.dose_unit_concept_id',\n",
|
|
|
- " u'drug_exposure.lot_number',\n",
|
|
|
- " u'drug_exposure.provider_id',\n",
|
|
|
- " u'drug_exposure.visit_occurrence_id',\n",
|
|
|
- " u'drug_exposure.drug_source_value',\n",
|
|
|
- " u'drug_exposure.drug_source_concept_id',\n",
|
|
|
- " u'drug_exposure.route_source_value',\n",
|
|
|
- " u'drug_exposure.dose_unit_source_value']"
|
|
|
+ "array([1, 3, 0, 0])"
|
|
|
]
|
|
|
},
|
|
|
- "execution_count": 111,
|
|
|
+ "execution_count": 6,
|
|
|
"metadata": {},
|
|
|
"output_type": "execute_result"
|
|
|
}
|
|
@@ -250,12 +246,7 @@
|
|
|
"#\n",
|
|
|
"# find every table with person id at the very least or a subset of fields\n",
|
|
|
"#\n",
|
|
|
- "info = get_tables(client,'raw',['person_id'])\n",
|
|
|
- "# get_fields(xo=names[0],xi=names[1:4],join='person_id')\n",
|
|
|
- "\n",
|
|
|
- "# q = ['person_id']\n",
|
|
|
- "# pairs = list(itertools.combinations(names,len(names)))\n",
|
|
|
- "# pairs[0]"
|
|
|
+ "np.random.randint(0,4,size=4)"
|
|
|
]
|
|
|
},
|
|
|
{
|
|
@@ -287,6 +278,72 @@
|
|
|
"x_ = 1"
|
|
|
]
|
|
|
},
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 10,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [],
|
|
|
+ "source": [
|
|
|
+ "x_ = pd.DataFrame({\"group\":[1,1,1,1,1], \"size\":[2,1,1,1,1]})"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "cell_type": "code",
|
|
|
+ "execution_count": 12,
|
|
|
+ "metadata": {},
|
|
|
+ "outputs": [
|
|
|
+ {
|
|
|
+ "data": {
|
|
|
+ "text/html": [
|
|
|
+ "<div>\n",
|
|
|
+ "<style scoped>\n",
|
|
|
+ " .dataframe tbody tr th:only-of-type {\n",
|
|
|
+ " vertical-align: middle;\n",
|
|
|
+ " }\n",
|
|
|
+ "\n",
|
|
|
+ " .dataframe tbody tr th {\n",
|
|
|
+ " vertical-align: top;\n",
|
|
|
+ " }\n",
|
|
|
+ "\n",
|
|
|
+ " .dataframe thead th {\n",
|
|
|
+ " text-align: right;\n",
|
|
|
+ " }\n",
|
|
|
+ "</style>\n",
|
|
|
+ "<table border=\"1\" class=\"dataframe\">\n",
|
|
|
+ " <thead>\n",
|
|
|
+ " <tr style=\"text-align: right;\">\n",
|
|
|
+ " <th></th>\n",
|
|
|
+ " <th>size</th>\n",
|
|
|
+ " </tr>\n",
|
|
|
+ " <tr>\n",
|
|
|
+ " <th>group</th>\n",
|
|
|
+ " <th></th>\n",
|
|
|
+ " </tr>\n",
|
|
|
+ " </thead>\n",
|
|
|
+ " <tbody>\n",
|
|
|
+ " <tr>\n",
|
|
|
+ " <th>1</th>\n",
|
|
|
+ " <td>1.2</td>\n",
|
|
|
+ " </tr>\n",
|
|
|
+ " </tbody>\n",
|
|
|
+ "</table>\n",
|
|
|
+ "</div>"
|
|
|
+ ],
|
|
|
+ "text/plain": [
|
|
|
+ " size\n",
|
|
|
+ "group \n",
|
|
|
+ "1 1.2"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ "execution_count": 12,
|
|
|
+ "metadata": {},
|
|
|
+ "output_type": "execute_result"
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "source": [
|
|
|
+ "x_.groupby(['group']).mean()\n"
|
|
|
+ ]
|
|
|
+ },
|
|
|
{
|
|
|
"cell_type": "code",
|
|
|
"execution_count": null,
|