\n"," \n"," | 0 | \n"," 83957610 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-17 | \n"," 1652850000 | \n"," 1652871600 | \n"," {'wakeupduration': 2779, 'wakeupcount': 2, 'du... | \n"," 1652871600 | \n"," 1652871600 | \n","
\n"," \n"," | 1 | \n"," 35325592 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-18 | \n"," 1652943600 | \n"," 1652961600 | \n"," {'wakeupduration': 858, 'wakeupcount': 1, 'dur... | \n"," 1652961600 | \n"," 1652961600 | \n","
\n"," \n"," | 2 | \n"," 88681721 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-19 | \n"," 1653026400 | \n"," 1653048000 | \n"," {'wakeupduration': 1519, 'wakeupcount': 0, 'du... | \n"," 1653048000 | \n"," 1653048000 | \n","
\n"," \n"," | 3 | \n"," 41466808 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-20 | \n"," 1653120000 | \n"," 1653134400 | \n"," {'wakeupduration': 1202, 'wakeupcount': 0, 'du... | \n"," 1653134400 | \n"," 1653134400 | \n","
\n"," \n"," | 4 | \n"," 46504192 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-21 | \n"," 1653210000 | \n"," 1653228000 | \n"," {'wakeupduration': 939, 'wakeupcount': 1, 'dur... | \n"," 1653228000 | \n"," 1653228000 | \n","
\n"," \n"," | 5 | \n"," 29254421 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-22 | \n"," 1653292800 | \n"," 1653314400 | \n"," {'wakeupduration': 489, 'wakeupcount': 0, 'dur... | \n"," 1653314400 | \n"," 1653314400 | \n","
\n"," \n"," | 6 | \n"," 77081597 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-23 | \n"," 1653361200 | \n"," 1653386400 | \n"," {'wakeupduration': 2876, 'wakeupcount': 0, 'du... | \n"," 1653386400 | \n"," 1653386400 | \n","
\n"," \n"," | 7 | \n"," 91453130 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-24 | \n"," 1653469200 | \n"," 1653483600 | \n"," {'wakeupduration': 1023, 'wakeupcount': 0, 'du... | \n"," 1653483600 | \n"," 1653483600 | \n","
\n"," \n"," | 8 | \n"," 80522091 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-25 | \n"," 1653544800 | \n"," 1653562800 | \n"," {'wakeupduration': 2442, 'wakeupcount': 0, 'du... | \n"," 1653562800 | \n"," 1653562800 | \n","
\n"," \n"," | 9 | \n"," 2513913 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-26 | \n"," 1653631200 | \n"," 1653645600 | \n"," {'wakeupduration': 865, 'wakeupcount': 1, 'dur... | \n"," 1653645600 | \n"," 1653645600 | \n","
\n"," \n"," | 10 | \n"," 40289961 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-27 | \n"," 1653724800 | \n"," 1653753600 | \n"," {'wakeupduration': 2142, 'wakeupcount': 1, 'du... | \n"," 1653753600 | \n"," 1653753600 | \n","
\n"," \n"," | 11 | \n"," 84795018 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-28 | \n"," 1653814800 | \n"," 1653832800 | \n"," {'wakeupduration': 2484, 'wakeupcount': 0, 'du... | \n"," 1653832800 | \n"," 1653832800 | \n","
\n"," \n"," | 12 | \n"," 86832247 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-29 | \n"," 1653897600 | \n"," 1653926400 | \n"," {'wakeupduration': 2682, 'wakeupcount': 1, 'du... | \n"," 1653926400 | \n"," 1653926400 | \n","
\n"," \n"," | 13 | \n"," 14215648 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-30 | \n"," 1653980400 | \n"," 1653994800 | \n"," {'wakeupduration': 2796, 'wakeupcount': 1, 'du... | \n"," 1653994800 | \n"," 1653994800 | \n","
\n"," \n"," | 14 | \n"," 31454868 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-05-31 | \n"," 1654052400 | \n"," 1654070400 | \n"," {'wakeupduration': 2892, 'wakeupcount': 0, 'du... | \n"," 1654070400 | \n"," 1654070400 | \n","
\n"," \n"," | 15 | \n"," 87489117 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-06-01 | \n"," 1654146000 | \n"," 1654174800 | \n"," {'wakeupduration': 1298, 'wakeupcount': 0, 'du... | \n"," 1654174800 | \n"," 1654174800 | \n","
\n"," \n"," | 16 | \n"," 83035559 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-06-02 | \n"," 1654243200 | \n"," 1654257600 | \n"," {'wakeupduration': 110, 'wakeupcount': 0, 'dur... | \n"," 1654257600 | \n"," 1654257600 | \n","
\n"," \n"," | 17 | \n"," 67312014 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-06-03 | \n"," 1654311600 | \n"," 1654329600 | \n"," {'wakeupduration': 2373, 'wakeupcount': 0, 'du... | \n"," 1654329600 | \n"," 1654329600 | \n","
\n"," \n"," | 18 | \n"," 43615706 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-06-04 | \n"," 1654401600 | \n"," 1654419600 | \n"," {'wakeupduration': 2688, 'wakeupcount': 2, 'du... | \n"," 1654419600 | \n"," 1654419600 | \n","
\n"," \n"," | 19 | \n"," 85920406 | \n"," America/Los_Angeles | \n"," 16 | \n"," 93 | \n"," d41d8cd98f00b204e9800998ecf8427e | \n"," 2022-06-05 | \n"," 1654502400 | \n"," 1654524000 | \n"," {'wakeupduration': 2813, 'wakeupcount': 1, 'du... | \n"," 1654524000 | \n"," 1654524000 | \n","
\n"," \n","![](\"https://imgur.com/3Z2W8HK.png\")
\n","\n","Ever wanted a smart watch with an EKG? Introducing the [Withings ScanWatch](https://www.withings.com/us/en/scanwatch), a smart watch that can not only detect your heart rate optically, but also with an EKG! While this notebook is meant for the ScanWatch, it can be easily adapted to any other Withings product with some modifications.\n","\n","Also note that you need to set up your Withings ScanWatch prior to running this notebook.\n","\n","If you want to know more about Withings Body+ Scale, see the [README](https://github.com/alrojo/wearipedia/tree/main/wearables/withings-scanwatch) for a detailed analysis of performances, sensors, data privacy, and extraction pipelines.\n","\n","We will be able to extract the following parameters (see the definitions at [this documenation page](https://developer.withings.com/api-reference#operation/sleepv2-getsummary)):\n","\n","![](\"https://i.imgur.com/ttWojjU.png\")
\n","\n","Now we can proceed with the rest of the notebook.\n","\n","To be able to make requests to the API and extract the data we need, we need to first issue an access token. This (ephemeral) access token will serve as our key to the data. While, you don't necessarily need to be familiar with how the issuing of the authtoken occurs, you can learn more about it by visiting [the official Withings tutorial](https://developer.withings.com/developer-guide/v3/integration-guide/public-health-data-api/get-access/oauth-web-flow/)."],"metadata":{"id":"d2Kbo6d4waj9"}},{"cell_type":"code","source":["#@title 6. Enter your credentials below (from the application you just created)\n","\n","# we hide a bunch of the library imports in here :)\n","!pip install -q july\n","\n","import requests\n","import urllib\n","import json\n","from datetime import datetime\n","from tqdm import tqdm\n","\n","import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","from scipy.ndimage import gaussian_filter\n","from scipy import stats\n","import july\n","from july.utils import date_range\n","\n","CLIENT_ID = \"\" #@param {type:\"string\"}\n","CUSTOMER_SECRET = \"\" #@param {type:\"string\"}\n","\n","STATE = 'string'\n","ACCOUNT_URL = 'https://account.withings.com'\n","CALLBACK_URI = 'https://wbsapi.withings.net/v2/oauth2'\n","\n","\n","payload = {'response_type': 'code', # imposed string by the api\n"," 'client_id': CLIENT_ID,\n"," 'state': STATE,\n"," 'scope': 'user.info,user.metrics,user.activity', # see docs (https://developer.withings.com/api-reference/#operation/oauth2-authorize) for enhanced scope\n"," 'redirect_uri': CALLBACK_URI, # URL of this app\n"," #'mode': 'demo' # Use demo mode, DELETE THIS FOR REAL APP\n","}\n","\n","url = f'{ACCOUNT_URL}/oauth2_user/authorize2?'\n","\n","for key, value in payload.items():\n"," url += f'{key}={value}&'\n","\n","url = url[:-1]\n","\n","print(url)"],"metadata":{"cellView":"form","id":"ev3-mu5ou88c","executionInfo":{"status":"ok","timestamp":1663815667013,"user_tz":240,"elapsed":6594,"user":{"displayName":"Rodrigo J Castellon","userId":"11221204973290737308"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"bbded490-c845-4e9d-924e-9ff08d0bc17c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["https://account.withings.com/oauth2_user/authorize2?response_type=code&client_id=&state=string&scope=user.info,user.metrics,user.activity&redirect_uri=https://wbsapi.withings.net/v2/oauth2\n"]}]},{"cell_type":"markdown","source":["7. Now visit the above URL and click \"Allow this app\", and copy the URL you were redirected to into the text field below. Note that if you mess up once, you have to go through the above URL again (including clicking \"Allow this app\"). Also, the URL is only valid for 30 seconds, so be quick!"],"metadata":{"id":"kpLZcG86vSuL"}},{"cell_type":"code","source":["#@title 7. Copy and paste the URL you were redirected to below\n","redirect_url = \"https://wbsapi.withings.net/v2/oauth2?code=09f095bcd678c7beaa9285e189c5a3917329ed7d&state=string\" #@param {type:\"string\"}\n","\n","try:\n"," code = urllib.parse.parse_qs(urllib.parse.urlparse(redirect_url).query)['code'][0]\n","except Exception as e:\n"," print(f'Caught error:\\n{e}\\n')\n"," print(\"Please copy and paste the entire URL (including https)\")\n","\n","params = {\n"," 'action': 'requesttoken',\n"," 'grant_type': 'authorization_code',\n"," 'client_id': CLIENT_ID,\n"," 'client_secret': CUSTOMER_SECRET,\n"," 'code': code,\n"," #'scope': 'user.info',\n"," 'redirect_uri': 'https://wbsapi.withings.net/v2/oauth2'\n","}\n","\n","out = requests.get('https://wbsapi.withings.net/v2/oauth2', data=params)\n","\n","out = json.loads(out.text)\n","\n","try:\n"," access_token = out['body']['access_token']\n","except KeyError as e:\n"," print('Took too long to paste in redirect URL. Please repeat step 7.')"],"metadata":{"id":"oZ8kzSuBoily","cellView":"form","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1663815667530,"user_tz":240,"elapsed":523,"user":{"displayName":"Rodrigo J Castellon","userId":"11221204973290737308"}},"outputId":"ae5981bf-92a5-4ade-949a-7aefac93cbf3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Took too long to paste in redirect URL. Please repeat step 7.\n"]}]},{"cell_type":"markdown","source":["Now that we have our access token, we can begin making requests to the API! This access token will last only three hours, though, so you would need to re-do step 7 if three hours pass."],"metadata":{"id":"JObtRZuFvuJl"}},{"cell_type":"markdown","source":["# 3. Data extraction\n","\n","Here, data extraction is pretty simple! We've made it possible to get all heart rate and sleep data in one function call each.\n","\n","If you need to customize this part further and need to dig into the code, notice that all we need to do is make a few GET requests with the right query parameters. See [the overall health data API page](https://developer.withings.com/developer-guide/v3/integration-guide/public-health-data-api/data-api/all-available-health-data) or the [\"Measure\" endpoints specifically](https://developer.withings.com/api-reference/#operation/measurev2-getactivity) for more info."],"metadata":{"id":"GUtc7x14ebLe"}},{"cell_type":"code","source":["#@title Enter start and end dates\n","start_date = \"2020-05-20\" #@param {type:\"date\"}\n","end_date = \"2022-07-20\" #@param {type:\"date\"}\n","\n","import requests\n","import urllib\n","import json\n","from datetime import datetime\n","from tqdm import tqdm\n","\n","import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","from scipy.ndimage import gaussian_filter\n","from scipy import stats\n","import july\n","from july.utils import date_range\n","\n","from datetime import timedelta, datetime\n","import pandas as pd\n","import numpy as np\n","\n","def create_synthetic_sleeps_df():\n","\n"," syn_sleeps = pd.DataFrame()\n","\n"," syn_sleeps['id'] = np.random.randint(0, 100000000, size=(20,))\n"," syn_sleeps['timezone'] = 'America/Los_Angeles'\n"," syn_sleeps['model'] = 16\n"," syn_sleeps['model_id'] = 93\n"," syn_sleeps['hash_deviceid'] = 'd41d8cd98f00b204e9800998ecf8427e' # randomly generated\n"," syn_sleeps['date'] = [datetime.fromtimestamp(datetime.strptime('2022-05-17', '%Y-%m-%d').timestamp() + i * 24 * 3600).strftime('%Y-%m-%d') for i in range(20)]\n","\n","\n"," startdates = []\n"," enddates = []\n","\n"," for date in syn_sleeps['date']:\n"," sleep_start = np.random.randint(20, 27)\n"," sleep_time = np.random.randint(4, 9)\n","\n"," startdate = datetime.strptime(date, '%Y-%m-%d') + timedelta(hours=sleep_start + 7)\n","\n"," enddate = startdate + timedelta(hours=sleep_time)\n","\n"," startdate, enddate = int(startdate.timestamp()), int(enddate.timestamp())\n","\n"," startdates.append(startdate)\n"," enddates.append(enddate)\n","\n"," all_data = []\n","\n"," for i in range(20):\n","\n"," data = {\n"," 'wakeupduration': np.random.randint(0, 3000),\n"," 'wakeupcount': np.random.poisson(1),\n"," 'durationtosleep': np.random.randint(120, 180),\n"," 'durationtowakeup': np.random.randint(0, 700),\n"," 'total_timeinbed': np.random.randint(10000, 50000),\n"," 'total_sleep_time': np.random.randint(10000, 50000),\n"," 'sleep_efficiency': np.random.rand() * .1 + .9,\n"," 'sleep_latency': np.random.randint(120, 130),\n"," 'wakeup_latency': np.random.randint(0, 800),\n"," 'waso': np.random.randint(0, 4000),\n"," 'nb_rem_episodes': 0,\n"," 'out_of_bed_count': 0,\n"," 'lightsleepduration': np.random.randint(6000, 35000),\n"," 'deepsleepduration': np.random.randint(3000, 17000),\n"," 'hr_average': np.random.randint(55, 65),\n"," 'hr_min': np.random.randint(40, 60),\n"," 'hr_max': np.random.randint(70, 120),\n"," 'sleep_score': np.random.randint(30, 80)\n","\n"," }\n","\n"," all_data.append(data)\n","\n"," syn_sleeps['startdate'] = startdates\n"," syn_sleeps['enddate'] = enddates\n","\n"," syn_sleeps['data'] = all_data\n","\n"," syn_sleeps['created'] = enddates\n"," syn_sleeps['modified'] = enddates\n","\n"," return syn_sleeps\n","\n","from tqdm.notebook import tqdm\n","import hashlib\n","\n","def create_syn_hr(syn_sleeps):\n"," start_day = datetime.strptime('2022-05-20', '%Y-%m-%d')\n","\n"," hour_usage = [0.8] * 3 + [0.9] * 7 + [1.0] * 10 + [0.9] * 4\n","\n"," datetimes = []\n","\n"," for day_offset in tqdm(range(20)):\n"," for hour_offset in range(24):\n"," for minute_offset in range(0, 60, 10):\n"," day = start_day + timedelta(days=day_offset)\n"," hour = day + timedelta(hours=hour_offset)\n"," minute = hour + timedelta(minutes=minute_offset)\n","\n"," if np.random.uniform(0, 1) < hour_usage[hour_offset]:\n"," datetimes.append(minute)\n","\n"," hr_measurements = (np.random.randn(len(datetimes)) * 5 + 90).astype('int')\n","\n"," timestamps = np.array([dt.timestamp() for dt in datetimes])\n","\n"," for i, (startdate, enddate) in tqdm(enumerate(zip(syn_sleeps.startdate, syn_sleeps.enddate))):\n"," idxes = np.where(np.logical_and(timestamps > startdate, timestamps < enddate))[0]\n"," duration = (enddate - startdate) / 3600\n"," avg_hr = -5/7 * duration + 64.1428571429\n","\n"," hr_measurements[idxes] = (np.random.randn(idxes.shape[0]) + avg_hr).astype('int')\n","\n"," syn_hr = pd.DataFrame()\n"," syn_hr['datetime'] = datetimes\n"," syn_hr['heart_rate'] = hr_measurements\n","\n"," syn_hr['model'] = 'ScanWatch'\n"," syn_hr['model_id'] = 93\n"," syn_hr['deviceid'] = hashlib.md5().hexdigest()\n","\n"," num_garbage = 1000\n","\n"," timestamps_garbage = np.random.uniform((start_day - timedelta(days=100)).timestamp(), start_day.timestamp(), size=num_garbage)\n","\n"," garbage_df = pd.DataFrame()\n"," garbage_df['datetime'] = [datetime.fromtimestamp(int(ts)) for ts in timestamps_garbage]\n"," garbage_df['heart_rate'] = (np.random.randn(num_garbage) * 5 + 90).astype('int')\n","\n"," garbage_df['model'] = None\n"," garbage_df['model_id'] = 1059\n"," garbage_df['deviceid'] = None\n","\n","\n"," syn_hr = pd.concat((garbage_df, syn_hr), ignore_index=True)\n","\n"," return syn_hr\n","\n","#########################################\n","# real code below, synthetic code above #\n","#########################################\n","\n","num_to_description = {1: 'Weight (kg)',\n"," 4: 'Height (meter)',\n"," 5: 'Fat Free Mass (kg)',\n"," 6: 'Fat Ratio (%)',\n"," 8: 'Fat Mass Weight (kg)',\n"," 9: 'Diastolic Blood Pressure (mmHg)',\n"," 10: 'Systolic Blood Pressure (mmHg)',\n"," 11: 'Heart Pulse (bpm) - only for BPM and scale devices',\n"," 12: 'Temperature (celsius)',\n"," 54: 'SP02 (%)',\n"," 71: 'Body Temperature (celsius)',\n"," 73: 'Skin Temperature (celsius)',\n"," 76: 'Muscle Mass (kg)',\n"," 77: 'Hydration (kg)',\n"," 88: 'Bone Mass (kg)',\n"," 91: 'Pulse Wave Velocity (m/s)',\n"," 123: 'VO2 max is a numerical measurement of your body’s ability to consume oxygen (ml/min/kg).',\n"," 135: 'QRS interval duration based on ECG signal',\n"," 136: 'PR interval duration based on ECG signal',\n"," 137: 'QT interval duration based on ECG signal',\n"," 138: 'Corrected QT interval duration based on ECG signal',\n"," 139: 'Atrial fibrillation result from PPG'}\n","\n","NUM_RETRIES = 3\n","\n","def fetch_all_wrapper(endpoint_url, data, headers, arr_key, parse_data=lambda x: x):\n"," # wrapper around public API that retrieves arbitrarily large # of\n"," # records, since there is a restriction of # of records per API response\n"," # NOTES:\n"," # out['body'][arr_key] is concatenated across several requests\n"," # parse_data is a function that parses the returned array\n","\n"," cur_offset = 0\n"," arr_complete = None\n","\n"," arr = None\n","\n"," while True:\n"," # endpoint can be flaky if the response payload is extremely large,\n"," # so retry at most NUM_RETRIES times\n"," for i in range(NUM_RETRIES):\n"," data_args = {\n"," **data,\n"," 'offset': cur_offset\n"," }\n","\n"," out = requests.post(endpoint_url, data=data_args, headers=headers)\n","\n"," out = json.loads(out.text)\n","\n"," if out['status'] == 401:\n"," raise Exception(f'request response is {out} for request {data_args} to endpoint {endpoint_url}, headers {headers}')\n","\n"," try:\n"," arr = parse_data(out['body'][arr_key])\n"," break\n"," except KeyError:\n"," if 'body' in out.keys():\n"," raise Exception(f'got key {arr_key}, expected one of {out[\"body\"].keys()}')\n"," elif out['status'] == 2555:\n"," # when the payload is too large, this is the status code\n"," continue\n"," else:\n"," raise Exception(f'request response is {out} for request {data_args} to endpoint {endpoint_url}, headers {headers}')\n","\n"," # for example, https://developer.withings.com/api-reference/#operation/measurev2-getactivity\n"," # vs. https://developer.withings.com/api-reference/#operation/measure-getmeas\n","\n"," if arr is not None:\n"," if type(arr) == type({}):\n"," if arr_complete is None:\n"," arr_complete = dict()\n","\n"," arr_complete.update(arr)\n","\n"," elif type(arr) == type([]):\n"," if arr_complete is None:\n"," arr_complete = []\n","\n"," arr_complete += arr\n","\n"," # continue if there's still more to get\n"," if 'more' in out['body'].keys() and out['body']['more'] == 1:\n"," cur_offset = out['body']['offset']\n"," else:\n"," break\n"," else:\n"," break\n","\n"," # replace with concatenated version\n"," if arr_complete is not None:\n"," out['body'][arr_key] = arr_complete\n","\n"," return out\n","\n","def fetch_all_heart_rate(start='2020-03-10', end='2022-05-28'):\n"," # get all dates heart rate was collected for\n"," out = fetch_all_wrapper('https://wbsapi.withings.net/v2/measure', {\n"," 'action': 'getactivity',\n"," 'startdateymd': start,\n"," 'enddateymd': end,\n"," 'data_fields': 'hr_average'\n"," }, {'Authorization': f'Bearer {access_token}'},\n"," arr_key='activities')\n","\n"," dates = [act['date'] for act in out['body']['activities']]\n","\n"," # now for each date get the heart rate data and store as list of dicts\n"," dict_list = []\n"," for date in tqdm(dates):\n"," out = fetch_all_wrapper('https://wbsapi.withings.net/v2/measure', {\n"," 'action': 'getintradayactivity',\n"," 'startdate': int(datetime.strptime(date, '%Y-%m-%d').timestamp()),\n"," 'enddate': int(datetime.strptime(date, '%Y-%m-%d').timestamp()) + 24 * 3600,\n"," 'data_fields': 'heart_rate'\n"," }, {'Authorization': f'Bearer {access_token}'},\n"," arr_key='series')\n","\n","\n"," if 'body' in out.keys():\n"," dict_list += [{'datetime': datetime.fromtimestamp(int(k)), **v} for k,v in out['body']['series'].items()]\n","\n"," df = pd.DataFrame.from_dict(dict_list)\n","\n"," return df\n","\n","\n","def fetch_all_sleeps(start='2020-03-10', end='2022-05-28'):\n"," out = fetch_all_wrapper('https://wbsapi.withings.net/v2/sleep', {\n"," 'action': 'getsummary',\n"," 'startdateymd': '2020-07-01',\n"," 'enddateymd': '2022-07-01',\n"," 'data_fields': 'nb_rem_episodes,sleep_efficiency,sleep_latency,total_sleep_time,total_timeinbed,wakeup_latency,waso,asleepduration,deepsleepduration,durationtosleep,durationtowakeup,hr_average,hr_max,hr_min,lightsleepduration,night_events,out_of_bed_count,remsleepduration,rr_average,rr_max,rr_min,sleep_score,snoring,snoringepisodecount,wakeupcount,wakeupduration'\n"," }, {'Authorization': f'Bearer {access_token}'}, arr_key='series')\n","\n"," df = pd.DataFrame.from_dict(out['body']['series'])\n","\n"," return df\n","\n","SEED = 3\n","\n","np.random.seed(SEED)"],"metadata":{"id":"oIhFUppKKwBC","cellView":"form"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["synthetic = True\n","\n","if synthetic:\n"," sleeps_df = create_synthetic_sleeps_df()\n"," hr_df = create_syn_hr(sleeps_df)\n","else:\n"," hr_df = fetch_all_heart_rate(start=start_date, end=end_date)\n"," sleeps_df = fetch_all_sleeps(start=start_date, end=end_date)"],"metadata":{"id":"b22VGbw0KIms","colab":{"base_uri":"https://localhost:8080/","height":81,"referenced_widgets":["930b7e70df9a42c4b282f5b8ecb92bc5","911bf58e0a1d4e2fb57f46e6b0668a3f","1711ac32959545bf9df96ba076057a76","276f65ccf4874dc48604504e8ef10e99","9e44355bf98348fbb850cd3b1e0f92aa","99386ad72c284b6289367c4b3cd99bc8","9ea4d29262ad4e09966001dc679c6479","44e09ccc57e44949b5325b075c40cb87","80df3276aba74a978ba14fa95ef8d36a","5b14f78c29e04907985b82334a96c1db","160406f93e98471bae7e2a9d3fe9c65f","31ef4e0b99bf4e11bea3bf8a6a8582cd","472153722daf42fbbd0be4ce62c9a12e","83b92c59fdda4e98bb69eeb13e020332","67ba32fce1294fa1b6bcd214c899dc7d","73952c4d26d749348fa3bcf45825fe8f","0a40a9cbff56441aa4ee13c7b236d950","c11e9961b91b4f569afe85f750cbca7f","a62f3e0e90c542ad808f739770e620b0","b289b8b49c8546a5a0d1cbdd25addbb4","c357242ffed449f3a11ca89c27a1c51d","fb8714e423984166b645789160bd52cb"]},"executionInfo":{"status":"ok","timestamp":1663815668059,"user_tz":240,"elapsed":319,"user":{"displayName":"Rodrigo J Castellon","userId":"11221204973290737308"}},"outputId":"00d75cbb-d717-438c-f545-a73b8b32c35d"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":[" 0%| | 0/20 [00:00\n"," ![](\"https://imgur.com/w7Z9G9X.png\")
\n","\n","*Above is a plot taken from the official Withings mobile app.*"],"metadata":{"id":"V-9IFYQAk9JZ"}},{"cell_type":"code","source":["#@title Enter date\n","date = \"2022-05-25\" #@param {type:\"date\"}\n","from matplotlib.ticker import FormatStrFormatter\n","import matplotlib.dates as dates\n","import matplotlib.transforms\n","\n","from dateutil import tz\n","from scipy.interpolate import make_interp_spline\n","from datetime import timedelta\n","\n","# measurements are taken every 10 minutes, displayed on app every 30\n","HEART_RATE_RECORDING_LENGTH = 30 * 60\n","\n","with plt.style.context('dark_background'):\n"," # get the start and end times as timestamps by using the datetime library\n"," start_ts = datetime.strptime(date + ' 00:00:00-07:00', '%Y-%m-%d %H:%M:%S%z').timestamp()\n"," #end_ts = datetime.strptime(date + ' 23:59:59-07:00', '%Y-%m-%d %H:%M:%S%z').timestamp()\n"," end_ts = start_ts + 24 * 3600 + 30 * 60 # 00:30 the next day\n"," #end_ts = datetime.strptime('2022-05-25 00:30:00-07:00', '%Y-%m-%d %H:%M:%S%z').timestamp()\n","\n"," # now find the indices in hr_df.timestamp that match as closely as possible\n"," start_idx = np.argmin(np.abs(hr_df.datetime.apply(lambda x: x.timestamp()) - start_ts))\n"," end_idx = np.argmin(np.abs(hr_df.datetime.apply(lambda x: x.timestamp()) - end_ts))\n","\n"," x = hr_df.datetime.iloc[start_idx:end_idx]\n"," y = hr_df.heart_rate.iloc[start_idx:end_idx]\n","\n"," # make it not as bumpy with gaussian filter. note that this does not reproduce\n"," # the curve exactly, as I'm not sure what smoothing algorithm they used...\n"," y = gaussian_filter(y, sigma=3)\n","\n"," fig = plt.figure(figsize=(18,6), facecolor='black')\n","\n"," x_timestamp = np.array([x_.timestamp() for x_ in x])\n","\n"," # get the gaps. we include [6] as well because when you do np.diff,\n"," # it actually leaves out exactly one element\n"," differences = np.concatenate((np.diff(x_timestamp), [60 * 10]))\n","\n"," # interpret a gap (i.e. when a user takes off the device for some prolonged\n"," # period of time) as any two measurements that are taken more than\n"," # 6 * 2 = 12 seconds apart, to account for minor variations around 6s\n"," gap_idxes = np.where(differences > HEART_RATE_RECORDING_LENGTH * 2)[0]\n","\n"," # get the sleeps\n"," sleep_idxes = []\n"," for lower, upper in zip(sleeps_df.startdate, sleeps_df.enddate):\n"," if lower < start_ts or upper > end_ts:\n"," continue\n"," # get the location in the timestamp array that is closest to `lower`\n"," lower_idx = np.argmin(np.abs((x_timestamp - lower) - 0))\n"," # get the location in the timestamp array that is closest to `upper`\n"," upper_idx = np.argmin(np.abs((x_timestamp - upper) - 0))\n","\n"," sleep_idxes.append((lower_idx, upper_idx))\n","\n"," # first, we just plot the entire thing\n"," plt.plot(x, y, linewidth=3, color='#85a6f7')\n","\n"," # now we overlay sleeps\n"," for sleep_start, sleep_end in sleep_idxes:\n"," plt.plot(x[sleep_start:sleep_end], y[sleep_start:sleep_end], linewidth=3, color='#3E414C')\n","\n"," plt.axvline(x=x.iloc[sleep_start], linestyle='--', linewidth=1.5, color='#2F303A')\n","\n"," plt.axvline(x=x.iloc[sleep_end], linestyle='--', linewidth=1.5, color='#2F303A')\n","\n"," # now we overlay gaps by overlaying with white\n"," for gap_idx in gap_idxes:\n"," plt.plot(x[gap_idx:gap_idx+2], y[gap_idx:gap_idx+2], linewidth=5, color='black')\n","\n"," plt.ylim(40, 180)\n"," plt.xlim(x.iloc[0], x.iloc[-1] - pd.Timedelta(minutes=30))\n","\n"," datetimes = [\n"," datetime.strptime(date + ' 00:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 04:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 08:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 12:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 16:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 20:00:00-0700', '%Y-%m-%d %H:%M:%S%z'),\n"," datetime.strptime(date + ' 00:00:00-0700', '%Y-%m-%d %H:%M:%S%z') + timedelta(days=1)\n"," ]\n","\n"," plt.xticks(ticks=datetimes, labels=['12AM', '4AM', '8AM', '12PM', '4PM', '8PM', '12AM'])\n","\n"," # get the y-axis ticks to appear on the right\n"," plt.gca().yaxis.tick_right()\n","\n"," #plt.gca().grid(axis='x', color='#2F303A')\n"," plt.gca().grid(axis='y', color='#2F303A', which='major')\n","\n"," # hide x-axis and make the xtick labels 16 size\n"," plt.tick_params(\n"," axis='x', # changes apply to the x-axis\n"," which='both', # both major and minor ticks are affected\n"," bottom=False, # ticks along the bottom edge are off\n"," labelsize=16,\n"," labelcolor='silver'\n"," )\n","\n"," # hide y-axis and make the ytick labels 16 size\n"," plt.tick_params(\n"," axis='y', # changes apply to the x-axis\n"," which='both', # both major and minor ticks are affected\n"," right=False, # ticks along the bottom edge are off\n"," labelsize=16,\n"," labelcolor='silver'\n"," )\n","\n"," # add offset to y-axis tick labels to make them appear above gridlines\n"," # instead of to the side\n"," # https://stackoverflow.com/questions/28615887/how-to-move-a-tick-label-in-matplotlib\n"," dx = -50/72.; dy = 15/72. \n"," offset = matplotlib.transforms.ScaledTranslation(dx, dy, fig.dpi_scale_trans)\n"," for label in plt.gca().yaxis.get_majorticklabels():\n"," label.set_transform(label.get_transform() + offset)\n","\n","\n"," title_text = datetime.strftime(datetime.strptime(date, '%Y-%m-%d'), '%A, %B %d')\n","\n"," plt.title(title_text, fontsize=25, pad=40)\n","\n"," # turn off all borders\n"," plt.gca().spines['left'].set_visible(False)\n"," plt.gca().spines['right'].set_visible(False)\n"," plt.gca().spines['top'].set_visible(False)\n"," plt.gca().spines['bottom'].set_visible(False)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":438},"executionInfo":{"status":"ok","timestamp":1663815668576,"user_tz":240,"elapsed":285,"user":{"displayName":"Rodrigo J Castellon","userId":"11221204973290737308"}},"outputId":"a3729da2-c7e4-40b0-bff3-76886d3e13c6","cellView":"form","id":"3pQu_Mor9CHX"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["![](\"https://imgur.com/tV3K4wB.png\")
\n","\n","*The above plot is taken directly from the mobile app.*"],"metadata":{"id":"JvSoAF9JntAl"}},{"cell_type":"code","source":["#@title Insert start of week (must be a Monday) and timezone name\n","\n","week_start = \"2022-05-23\" #@param {type:\"date\"}\n","timezone_name = \"America/Los_Angeles\" #@param {type:\"string\"}\n","\n","from matplotlib.ticker import FormatStrFormatter\n","import matplotlib.dates as dates\n","import matplotlib.transforms\n","import matplotlib.patches as patches\n","from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,\n"," AutoMinorLocator)\n","\n","from dateutil import tz\n","from scipy.interpolate import make_interp_spline\n","\n","from datetime import timezone\n","import pytz\n","\n","def timestamp_to_hour_min(timestamp, tz_name='America/Los_Angeles'):\n"," # convert timezone to '%H:%M' in local timezone\n","\n"," local_tz = pytz.timezone(tz_name)\n"," return datetime.fromtimestamp(int(timestamp)).replace(tzinfo=timezone.utc).astimezone(local_tz).strftime('%H:%M')\n","\n","def hour_min_to_vert_pos(hour_min):\n"," num_mins = (int(hour_min.split(':')[0]) * 60 + int(hour_min.split(':')[1]))\n"," vert_dist = (16 * 60 - num_mins) / (16 * 60)\n","\n"," return vert_dist\n","\n","# measurements are taken every 10 minutes, displayed on app every 30\n","HEART_RATE_RECORDING_LENGTH = 30 * 60\n","\n","NUM_DAYS = 7\n","RECT_WIDTH = 0.02\n","RECTS_START = 0.1\n","RECTS_END = 0.9\n","\n","# add on timezone\n","timezone_offset = datetime.now(pytz.timezone(timezone_name)).strftime('%z')\n","week_start = week_start + timezone_offset\n","week_start_ts = datetime.strptime(week_start, '%Y-%m-%d%z').timestamp()\n","\n","with plt.style.context('dark_background'):\n"," fig, ax = plt.subplots(figsize=(18,6), facecolor='black')\n","\n"," for day_num in range(NUM_DAYS):\n"," rect_center_pos = RECTS_START + day_num / (NUM_DAYS-1) * (RECTS_END - RECTS_START)\n"," # zorder to ensure it is drawn behind rect\n"," plt.axvline(x=rect_center_pos, linestyle='--', linewidth=1.5, color='#2F303A', zorder=0)\n","\n"," # rounded rectangles\n"," # https://stackoverflow.com/questions/58425392/bar-chart-with-rounded-corners-in-matplotlib\n","\n"," # subtract 4 hours because we want to capture sleeps that start from 8pm\n"," # and onward\n"," day_ts = week_start_ts + day_num * 24 * 3600 - 4 * 3600\n","\n"," row = sleeps_df[np.logical_and(sleeps_df.startdate >= day_ts, sleeps_df.enddate <= day_ts + 24 * 3600)]\n","\n"," # get the one with the longest sleep, this is how withings does it\n"," # source: https://support.withings.com/hc/en-us/community/posts/360026177173-Naps-don-t-count-for-sleep-tracking\n"," try:\n"," row = row.iloc[np.argmax(row.enddate - row.startdate)]\n"," except ValueError:\n"," continue\n","\n"," # do a shift by 4 hours to account for difference b/w\n"," # 8PM and 12AM, a bit of a hack\n"," row.startdate += 4 * 3600\n"," row.enddate += 4 * 3600\n","\n"," start, end = timestamp_to_hour_min(row.startdate, tz_name=timezone_name), timestamp_to_hour_min(row.enddate, tz_name=timezone_name)\n","\n"," top_pos, bottom_pos = hour_min_to_vert_pos(start), hour_min_to_vert_pos(end)\n","\n"," if row.data['sleep_score'] < 50:\n"," color = '#ff7455'\n"," elif row.data['sleep_score'] < 75:\n"," color = '#FEDF00'\n"," else:\n"," color = '#24ffa4'\n","\n"," rect = patches.FancyBboxPatch((rect_center_pos - RECT_WIDTH / 2, bottom_pos),\n"," RECT_WIDTH, top_pos - bottom_pos,\n"," boxstyle=\"round,pad=-0.0040,rounding_size=0.015\",\n"," linewidth=3, fc=color, ec='none')\n","\n"," ax.add_patch(rect)\n","\n"," plt.xticks(ticks=np.linspace(0.1,0.9,NUM_DAYS), labels=['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'])\n"," plt.yticks(ticks=np.linspace(0, 1, 5), labels=['12PM', '8AM', '4AM', '12AM', '8PM'])\n","\n"," # get the y-axis ticks to appear on the right\n"," plt.gca().yaxis.tick_right()\n","\n"," plt.gca().yaxis.set_minor_locator(AutoMinorLocator(n=2))\n"," plt.gca().grid(axis='y', color='#2F303A', which='both')\n","\n"," # hide x-axis and make the xtick labels 16 size\n"," plt.tick_params(\n"," axis='x', # changes apply to the x-axis\n"," which='both', # both major and minor ticks are affected\n"," bottom=False, # ticks along the bottom edge are off\n"," labelsize=16,\n"," labelcolor='silver'\n"," )\n","\n"," # hide y-axis and make the ytick labels 16 size\n"," plt.tick_params(\n"," axis='y', # changes apply to the x-axis\n"," which='both', # both major and minor ticks are affected\n"," right=False, # ticks along the bottom edge are off\n"," labelsize=16,\n"," labelcolor='silver'\n"," )\n","\n"," # add offset to y-axis tick labels to make them appear above gridlines\n"," # instead of to the side\n"," # https://stackoverflow.com/questions/28615887/how-to-move-a-tick-label-in-matplotlib\n"," dx = -50/72.; dy = 15/72. \n"," offset = matplotlib.transforms.ScaledTranslation(dx, dy, fig.dpi_scale_trans)\n"," for label in plt.gca().yaxis.get_majorticklabels():\n"," label.set_transform(label.get_transform() + offset)\n","\n"," start_day_fmted = datetime.strftime(datetime.strptime(week_start, '%Y-%m-%d%z'), '%b %d')\n"," end_day_fmted = datetime.strftime(datetime.strptime(week_start, '%Y-%m-%d%z') + timedelta(days=6), '%b %d')\n"," plt.title(f'{start_day_fmted} - {end_day_fmted}', size=20, pad=40)\n","\n"," plt.gca().set_axisbelow(True)\n"," # turn off all borders\n"," plt.gca().spines['left'].set_visible(False)\n"," plt.gca().spines['right'].set_visible(False)\n"," plt.gca().spines['top'].set_visible(False)\n"," plt.gca().spines['bottom'].set_visible(False)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":434},"id":"rQvaa5iHyNpT","executionInfo":{"status":"ok","timestamp":1663815669098,"user_tz":240,"elapsed":527,"user":{"displayName":"Rodrigo J Castellon","userId":"11221204973290737308"}},"outputId":"3cb2fd20-5828-4408-b32d-3d97b863e20a","cellView":"form"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["