{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "59d95960-2c42-4979-bffb-fc4e95f7570b", "metadata": {}, "outputs": [], "source": [ "import polars as pl" ] }, { "cell_type": "code", "execution_count": 3, "id": "31265a1f-414d-4a0c-b4fb-5a33aceb65af", "metadata": {}, "outputs": [], "source": [ "# df = pl.read_csv('test_out_big.csv', ignore_errors=True)\n", "df = pl.read_csv('parsed_workerlogs_2024_12_16.txt', ignore_errors=True)\n", "df = df.with_columns(pl.col('datetime').str.to_datetime())\n", "res = df.filter(pl.col('msg') == 'received').select(pl.col('taskid'), pl.col('datetime')).join(df.filter(pl.col('msg') == 'succeeded'), on=\"taskid\", how=\"left\").select('taskid', start_time=pl.col('datetime'), success_time=pl.col('datetime_right'), exectime=pl.col('msgtime'), workerid=pl.col('workerid'), tag=pl.col('tag'), taskname=pl.col('taskname')).drop_nulls().with_columns(localworkerid=pl.col('tag').str.extract(r\"INFO\\/ForkPoolWorker-(\\d+)\", group_index=1)).select(pl.exclude('tag'))" ] }, { "cell_type": "code", "execution_count": 4, "id": "e1537299-358c-4293-bafc-0d13e5d81b15", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "[{'taskname': 're_restapi.libs.reports.tasks.send_health_report',\n", " 'exectime_min': 238.78,\n", " 'exectime_max': 238.78,\n", " 'exectime_mean': 238.78,\n", " 'exectime_median': 238.78,\n", " 'exectime_std': None,\n", " 'exectime_q25': 238.78,\n", " 'exectime_q75': 238.78},\n", " {'taskname': 're_restapi.libs.ipsum.ipsum_drop_powertable',\n", " 'exectime_min': 231.477,\n", " 'exectime_max': 231.477,\n", " 'exectime_mean': 231.477,\n", " 'exectime_median': 231.477,\n", " 'exectime_std': None,\n", " 'exectime_q25': 231.477,\n", " 'exectime_q75': 231.477},\n", " {'taskname': 'gireve.libs.locations.push_evse_avail_task',\n", " 'exectime_min': 0.13329,\n", " 'exectime_max': 50.8969,\n", " 'exectime_mean': 1.6022709375000013,\n", " 'exectime_median': 0.33353,\n", " 'exectime_std': 3.8949635523673067,\n", " 'exectime_q25': 0.23291,\n", " 'exectime_q75': 1.21161},\n", " {'taskname': 'gireve.libs.locations.push_park_task',\n", " 'exectime_min': 0.14699,\n", " 'exectime_max': 40.0796,\n", " 'exectime_mean': 1.4115833469665957,\n", " 'exectime_median': 0.37479,\n", " 'exectime_std': 3.0636598001555537,\n", " 'exectime_q25': 0.23828,\n", " 'exectime_q75': 1.1391},\n", " {'taskname': 'gireve.libs.storeandforward.gireve_retry_check_and_run',\n", " 'exectime_min': 0.02169,\n", " 'exectime_max': 33.3444,\n", " 'exectime_mean': 0.060002904794996495,\n", " 'exectime_median': 0.03152,\n", " 'exectime_std': 0.8783807205369593,\n", " 'exectime_q25': 0.02815,\n", " 'exectime_q75': 0.03772},\n", " {'taskname': 'gireve.libs.hooks.cs_post_save.cs_post_save_task',\n", " 'exectime_min': 0.17908,\n", " 'exectime_max': 33.2225,\n", " 'exectime_mean': 0.7425301901565985,\n", " 'exectime_median': 0.391695,\n", " 'exectime_std': 1.6621227072699707,\n", " 'exectime_q25': 0.28003,\n", " 'exectime_q75': 0.68327},\n", " {'taskname': 're_restapi.libs.advenir.advenir_send_charge',\n", " 'exectime_min': 0.31333,\n", " 'exectime_max': 30.0134,\n", " 'exectime_mean': 4.874016326767094,\n", " 'exectime_median': 2.89043,\n", " 'exectime_std': 5.690269620567604,\n", " 'exectime_q25': 1.5812,\n", " 'exectime_q75': 5.84856},\n", " {'taskname': 're_restapi.libs.ipsum.send_single_ipsum_lastpower',\n", " 'exectime_min': 0.04605,\n", " 'exectime_max': 11.8619,\n", " 'exectime_mean': 0.1328267360339938,\n", " 'exectime_median': 0.09599,\n", " 'exectime_std': 0.2321972956814589,\n", " 'exectime_q25': 0.0809,\n", " 'exectime_q75': 0.12498},\n", " {'taskname': 're_restapi.libs.postpower.task_post_power_bytecode',\n", " 'exectime_min': 0.06397,\n", " 'exectime_max': 8.7991,\n", " 'exectime_mean': 0.197694960384119,\n", " 'exectime_median': 0.14089,\n", " 'exectime_std': 0.23314194745745556,\n", " 'exectime_q25': 0.11162,\n", " 'exectime_q75': 0.20175},\n", " {'taskname': 're_restapi.libs.postpower.task_tcp_post_power_bytecode',\n", " 'exectime_min': 0.06057,\n", " 'exectime_max': 7.64952,\n", " 'exectime_mean': 0.195499678533702,\n", " 'exectime_median': 0.14348,\n", " 'exectime_std': 0.22821424064981857,\n", " 'exectime_q25': 0.11079,\n", " 'exectime_q75': 0.2029},\n", " {'taskname': 're_restapi.libs.orders.send_recovery_state',\n", " 'exectime_min': 0.04109,\n", " 'exectime_max': 5.26535,\n", " 'exectime_mean': 0.09394207694532807,\n", " 'exectime_median': 0.0712,\n", " 'exectime_std': 0.156540945241742,\n", " 'exectime_q25': 0.06064,\n", " 'exectime_q75': 0.08991},\n", " {'taskname': 're_restapi.libs.orders.send_resume',\n", " 'exectime_min': 0.00928,\n", " 'exectime_max': 4.70691,\n", " 'exectime_mean': 0.057368659574468094,\n", " 'exectime_median': 0.02499,\n", " 'exectime_std': 0.2999546449427308,\n", " 'exectime_q25': 0.01828,\n", " 'exectime_q75': 0.04059},\n", " {'taskname': 're_restapi.libs.metrics.task_process_metric_message',\n", " 'exectime_min': 0.03438,\n", " 'exectime_max': 4.1262,\n", " 'exectime_mean': 0.10684756096420905,\n", " 'exectime_median': 0.07821,\n", " 'exectime_std': 0.12391577600619175,\n", " 'exectime_q25': 0.0648,\n", " 'exectime_q75': 0.10252},\n", " {'taskname': 're_restapi.libs.ipsum.task_post_ipsum_bytecode',\n", " 'exectime_min': 0.02878,\n", " 'exectime_max': 4.07619,\n", " 'exectime_mean': 0.06275795996442927,\n", " 'exectime_median': 0.04897,\n", " 'exectime_std': 0.062289887941839185,\n", " 'exectime_q25': 0.04335,\n", " 'exectime_q75': 0.06053},\n", " {'taskname': 're_restapi.libs.orders.send_order_off',\n", " 'exectime_min': 0.04214,\n", " 'exectime_max': 2.87086,\n", " 'exectime_mean': 0.08923561611374418,\n", " 'exectime_median': 0.07047,\n", " 'exectime_std': 0.10872877763325738,\n", " 'exectime_q25': 0.05891,\n", " 'exectime_q75': 0.08973},\n", " {'taskname': 're_restapi.libs.stripewebhook.sh_payment_intent_succeeded',\n", " 'exectime_min': 0.51218,\n", " 'exectime_max': 2.8165,\n", " 'exectime_mean': 0.9926075000000001,\n", " 'exectime_median': 0.714055,\n", " 'exectime_std': 0.5267428561713412,\n", " 'exectime_q25': 0.63767,\n", " 'exectime_q75': 1.11721},\n", " {'taskname': 'gireve.libs.sessions.push_session_task',\n", " 'exectime_min': 0.36118,\n", " 'exectime_max': 2.49733,\n", " 'exectime_mean': 0.7068926470588226,\n", " 'exectime_median': 0.502035,\n", " 'exectime_std': 0.38570995845027806,\n", " 'exectime_q25': 0.44371,\n", " 'exectime_q75': 1.12166},\n", " {'taskname': 're_restapi.libs.orders.send_order_on',\n", " 'exectime_min': 0.04226,\n", " 'exectime_max': 1.93464,\n", " 'exectime_mean': 0.08757706088992974,\n", " 'exectime_median': 0.0683,\n", " 'exectime_std': 0.09386564267329077,\n", " 'exectime_q25': 0.05725,\n", " 'exectime_q75': 0.09136},\n", " {'taskname': 're_restapi.libs.pilotage.task_process_message',\n", " 'exectime_min': 0.00105,\n", " 'exectime_max': 1.67776,\n", " 'exectime_mean': 0.024481181405895683,\n", " 'exectime_median': 0.00184,\n", " 'exectime_std': 0.06763372234081201,\n", " 'exectime_q25': 0.00161,\n", " 'exectime_q75': 0.00559},\n", " {'taskname': 're_restapi.libs.user.tasks.update_expired_user',\n", " 'exectime_min': 0.03933,\n", " 'exectime_max': 1.58621,\n", " 'exectime_mean': 0.06267312993039457,\n", " 'exectime_median': 0.056145,\n", " 'exectime_std': 0.0350771373047798,\n", " 'exectime_q25': 0.05067,\n", " 'exectime_q75': 0.06325},\n", " {'taskname': 're_restapi.libs.ipsum.ipsum_powertable_scansave',\n", " 'exectime_min': 0.03568,\n", " 'exectime_max': 1.49204,\n", " 'exectime_mean': 0.07203673201856149,\n", " 'exectime_median': 0.05624,\n", " 'exectime_std': 0.0703718774358662,\n", " 'exectime_q25': 0.04999,\n", " 'exectime_q75': 0.06854},\n", " {'taskname': 'gireve.libs.commands.start_session',\n", " 'exectime_min': 1.48396,\n", " 'exectime_max': 1.48396,\n", " 'exectime_mean': 1.48396,\n", " 'exectime_median': 1.48396,\n", " 'exectime_std': None,\n", " 'exectime_q25': 1.48396,\n", " 'exectime_q75': 1.48396},\n", " {'taskname': 're_restapi.libs.postpower.park_powertable_scansave',\n", " 'exectime_min': 0.02959,\n", " 'exectime_max': 1.45707,\n", " 'exectime_mean': 0.055580026682134555,\n", " 'exectime_median': 0.044495,\n", " 'exectime_std': 0.052481496202111545,\n", " 'exectime_q25': 0.03983,\n", " 'exectime_q75': 0.05337},\n", " {'taskname': 're_restapi.libs.postpower.task_post_power_legacy',\n", " 'exectime_min': 0.05043,\n", " 'exectime_max': 0.97348,\n", " 'exectime_mean': 0.08775651037344391,\n", " 'exectime_median': 0.07291500000000001,\n", " 'exectime_std': 0.06111770958543688,\n", " 'exectime_q25': 0.06582,\n", " 'exectime_q75': 0.0865},\n", " {'taskname': 're_restapi.libs.charge.close_charge_for_offline_parks',\n", " 'exectime_min': 0.0802,\n", " 'exectime_max': 0.86007,\n", " 'exectime_mean': 0.15660583333333333,\n", " 'exectime_median': 0.11333,\n", " 'exectime_std': 0.14527895568611524,\n", " 'exectime_q25': 0.09637,\n", " 'exectime_q75': 0.13306},\n", " {'taskname': 'celery.backend_cleanup',\n", " 'exectime_min': 0.61073,\n", " 'exectime_max': 0.61073,\n", " 'exectime_mean': 0.61073,\n", " 'exectime_median': 0.61073,\n", " 'exectime_std': None,\n", " 'exectime_q25': 0.61073,\n", " 'exectime_q75': 0.61073},\n", " {'taskname': 're_restapi.libs.orders.send_suspend',\n", " 'exectime_min': 0.01004,\n", " 'exectime_max': 0.20127,\n", " 'exectime_mean': 0.033789275362318814,\n", " 'exectime_median': 0.02434,\n", " 'exectime_std': 0.02707346508510639,\n", " 'exectime_q25': 0.01865,\n", " 'exectime_q75': 0.03712},\n", " {'taskname': 'gireve.libs.commands.reply_accepted',\n", " 'exectime_min': 0.19888,\n", " 'exectime_max': 0.19888,\n", " 'exectime_mean': 0.19888,\n", " 'exectime_median': 0.19888,\n", " 'exectime_std': None,\n", " 'exectime_q25': 0.19888,\n", " 'exectime_q75': 0.19888},\n", " {'taskname': 're_restapi.libs.user.sessions.clear_expired_sessions',\n", " 'exectime_min': 0.07128,\n", " 'exectime_max': 0.07128,\n", " 'exectime_mean': 0.07128,\n", " 'exectime_median': 0.07128,\n", " 'exectime_std': None,\n", " 'exectime_q25': 0.07128,\n", " 'exectime_q75': 0.07128},\n", " {'taskname': 're_restapi.libs.confbackup.task_post_confbackup',\n", " 'exectime_min': 0.0552,\n", " 'exectime_max': 0.06847,\n", " 'exectime_mean': 0.061835,\n", " 'exectime_median': 0.061835,\n", " 'exectime_std': 0.009383306986345489,\n", " 'exectime_q25': 0.0552,\n", " 'exectime_q75': 0.06847}]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "exectime_stats = (\n", " res.group_by('taskname')\n", " .agg(\n", " pl.col('exectime').min().alias('exectime_min'),\n", " pl.col('exectime').max().alias('exectime_max'),\n", " pl.col('exectime').mean().alias('exectime_mean'),\n", " pl.col('exectime').median().alias('exectime_median'),\n", " pl.col('exectime').std().alias('exectime_std'),\n", " pl.col('exectime').quantile(0.25).alias('exectime_q25'),\n", " pl.col('exectime').quantile(0.75).alias('exectime_q75'),\n", " )\n", " .sort('taskname')\n", ")\n", "exectime_stats.sort('exectime_max').reverse().to_dicts()" ] }, { "cell_type": "code", "execution_count": 5, "id": "9db82a99-b1e4-4140-a04d-b1ee004b518c", "metadata": {}, "outputs": [], "source": [ "import altair as alt\n", "import plotly.graph_objects as go" ] }, { "cell_type": "code", "execution_count": 6, "id": "14864cd6-75bf-4678-99cc-eb651f5532df", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.LayerChart(...)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "error_bars = alt.Chart(exectime_stats).mark_errorbar(extent='stdev').encode(\n", " xError=alt.XError('exectime_std'),\n", " x=alt.X('exectime_mean'),\n", " y=alt.Y('taskname'),\n", ")\n", "points = alt.Chart(exectime_stats).mark_point(filled=True, color='black').encode(\n", " x=alt.X('exectime_mean'),\n", " y=alt.Y('taskname'),\n", ")\n", "error_bars + points" ] }, { "cell_type": "code", "execution_count": 7, "id": "b0538b9d-d613-4e55-afb9-3b6b874e54e0", "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.plotly.v1+json": { "config": { "plotlyServerURL": "https://plot.ly" }, "data": [ { "lowerfence": [ 0.61073, 0.19888, 1.48396, 0.17908, 0.13329, 0.14699, 0.36118, 0.02169, 0.31333, 0.0802, 0.0552, 231.477, 0.03568, 0.04605, 0.02878, 0.03438, 0.04214, 0.04226, 0.04109, 0.00928, 0.01004, 0.00105, 0.02959, 0.06397, 0.05043, 0.06057, 238.78, 0.51218, 0.07128, 0.03933 ], "median": [ 0.61073, 0.19888, 1.48396, 0.391695, 0.33353, 0.37479, 0.502035, 0.03152, 2.89043, 0.11333, 0.061835, 231.477, 0.05624, 0.09599, 0.04897, 0.07821, 0.07047, 0.0683, 0.0712, 0.02499, 0.02434, 0.00184, 0.044495, 0.14089, 0.07291500000000001, 0.14348, 238.78, 0.714055, 0.07128, 0.056145 ], "name": "Precompiled Quartiles", "q1": [ 0.61073, 0.19888, 1.48396, 0.28003, 0.23291, 0.23828, 0.44371, 0.02815, 1.5812, 0.09637, 0.0552, 231.477, 0.04999, 0.0809, 0.04335, 0.0648, 0.05891, 0.05725, 0.06064, 0.01828, 0.01865, 0.00161, 0.03983, 0.11162, 0.06582, 0.11079, 238.78, 0.63767, 0.07128, 0.05067 ], "q3": [ 0.61073, 0.19888, 1.48396, 0.68327, 1.21161, 1.1391, 1.12166, 0.03772, 5.84856, 0.13306, 0.06847, 231.477, 0.06854, 0.12498, 0.06053, 0.10252, 0.08973, 0.09136, 0.08991, 0.04059, 0.03712, 0.00559, 0.05337, 0.20175, 0.0865, 0.2029, 238.78, 1.11721, 0.07128, 0.06325 ], "type": "box", "upperfence": [ 0.61073, 0.19888, 1.48396, 33.2225, 50.8969, 40.0796, 2.49733, 33.3444, 30.0134, 0.86007, 0.06847, 231.477, 1.49204, 11.8619, 4.07619, 4.1262, 2.87086, 1.93464, 5.26535, 4.70691, 0.20127, 1.67776, 1.45707, 8.7991, 0.97348, 7.64952, 238.78, 2.8165, 0.07128, 1.58621 ], "x": [ "celery.backend_cleanup", "gireve.libs.commands.reply_accepted", "gireve.libs.commands.start_session", "gireve.libs.hooks.cs_post_save.cs_post_save_task", "gireve.libs.locations.push_evse_avail_task", "gireve.libs.locations.push_park_task", "gireve.libs.sessions.push_session_task", "gireve.libs.storeandforward.gireve_retry_check_and_run", "re_restapi.libs.advenir.advenir_send_charge", "re_restapi.libs.charge.close_charge_for_offline_parks", "re_restapi.libs.confbackup.task_post_confbackup", "re_restapi.libs.ipsum.ipsum_drop_powertable", "re_restapi.libs.ipsum.ipsum_powertable_scansave", "re_restapi.libs.ipsum.send_single_ipsum_lastpower", "re_restapi.libs.ipsum.task_post_ipsum_bytecode", "re_restapi.libs.metrics.task_process_metric_message", "re_restapi.libs.orders.send_order_off", "re_restapi.libs.orders.send_order_on", "re_restapi.libs.orders.send_recovery_state", "re_restapi.libs.orders.send_resume", "re_restapi.libs.orders.send_suspend", "re_restapi.libs.pilotage.task_process_message", "re_restapi.libs.postpower.park_powertable_scansave", "re_restapi.libs.postpower.task_post_power_bytecode", "re_restapi.libs.postpower.task_post_power_legacy", "re_restapi.libs.postpower.task_tcp_post_power_bytecode", "re_restapi.libs.reports.tasks.send_health_report", "re_restapi.libs.stripewebhook.sh_payment_intent_succeeded", "re_restapi.libs.user.sessions.clear_expired_sessions", "re_restapi.libs.user.tasks.update_expired_user" ] } ], "layout": { "height": 800, "template": { "data": { "bar": [ { "error_x": { "color": "#2a3f5f" }, "error_y": { "color": "#2a3f5f" }, "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "bar" } ], "barpolar": [ { "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "barpolar" } ], "carpet": [ { "aaxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "baxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "type": "carpet" } ], "choropleth": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "choropleth" } ], "contour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "contour" } ], "contourcarpet": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "contourcarpet" } ], "heatmap": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "heatmap" } ], "heatmapgl": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "heatmapgl" } ], "histogram": [ { "marker": { "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "histogram" } ], "histogram2d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "histogram2d" } ], "histogram2dcontour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "histogram2dcontour" } ], "mesh3d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "mesh3d" } ], "parcoords": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "parcoords" } ], "pie": [ { "automargin": true, "type": "pie" } ], "scatter": [ { "fillpattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 }, "type": "scatter" } ], "scatter3d": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatter3d" } ], "scattercarpet": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattercarpet" } ], "scattergeo": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergeo" } ], "scattergl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergl" } ], "scattermapbox": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattermapbox" } ], "scatterpolar": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolar" } ], "scatterpolargl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolargl" } ], "scatterternary": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterternary" } ], "surface": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "type": "surface" } ], "table": [ { "cells": { "fill": { "color": "#EBF0F8" }, "line": { "color": "white" } }, "header": { "fill": { "color": "#C8D4E3" }, "line": { "color": "white" } }, "type": "table" } ] }, "layout": { "annotationdefaults": { "arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1 }, "autotypenumbers": "strict", "coloraxis": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "colorscale": { "diverging": [ [ 0, "#8e0152" ], [ 0.1, "#c51b7d" ], [ 0.2, "#de77ae" ], [ 0.3, "#f1b6da" ], [ 0.4, "#fde0ef" ], [ 0.5, "#f7f7f7" ], [ 0.6, "#e6f5d0" ], [ 0.7, "#b8e186" ], [ 0.8, "#7fbc41" ], [ 0.9, "#4d9221" ], [ 1, "#276419" ] ], "sequential": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ], "sequentialminus": [ [ 0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1, "#f0f921" ] ] }, "colorway": [ "#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52" ], "font": { "color": "#2a3f5f" }, "geo": { "bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white" }, "hoverlabel": { "align": "left" }, "hovermode": "closest", "mapbox": { "style": "light" }, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": { "angularaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "radialaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "scene": { "xaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "yaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "zaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" } }, "shapedefaults": { "line": { "color": "#2a3f5f" } }, "ternary": { "aaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "baxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "caxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "title": { "x": 0.05 }, "xaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 }, "yaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 } } }, "yaxis": { "type": "log" } } }, "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "fig = go.Figure()\n", "fig.add_trace(go.Box(q1=exectime_stats['exectime_q25'], median=exectime_stats['exectime_median'],\n", " q3=exectime_stats['exectime_q75'], lowerfence=exectime_stats['exectime_min'],\n", " upperfence=exectime_stats['exectime_max'], name=\"Precompiled Quartiles\", x=exectime_stats['taskname']))\n", "fig.update_layout(height=800, yaxis={'type': 'log'})\n", "fig.show()" ] }, { "cell_type": "code", "execution_count": 8, "id": "93d97002-f06f-4878-a32d-8fc0f4fedfcd", "metadata": {}, "outputs": [], "source": [ "# Use-percentage dataframe\n", "updf = res.with_columns(worker=pl.concat_str(['workerid','localworkerid'], separator='-')).select(pl.exclude(\"workerid\",\"localworkerid\"))" ] }, { "cell_type": "code", "execution_count": 9, "id": "4049902a-01ae-4c2f-9b9d-7f6e19fb20d2", "metadata": {}, "outputs": [], "source": [ "from datetime import datetime, timedelta\n", "from zoneinfo import ZoneInfo\n", "thetime = pl.datetime_range(\n", " start=updf.get_column('start_time').min(),\n", " end=updf.get_column('start_time').max(),\n", " # start=datetime(2024,12,16,0,0,0,tzinfo=ZoneInfo('Europe/Rome')).astimezone(ZoneInfo('UTC')),\n", " # end=datetime(2024,12,17,0,0,0,tzinfo=ZoneInfo('Europe/Rome')).astimezone(ZoneInfo('UTC')),\n", " interval=\"1m\",\n", " eager=True,\n", ").alias('time').to_frame()\n", "thetime = thetime.with_columns(time2=pl.col('time')+timedelta(minutes=1))\n", "# def interval_intersect(a,b,c,d):\n", "# \"\"\"\n", "# a,b intersect c,d\n", "# the input are column names\n", "# real formula: (p1 and p2) or (p3 and p4)\n", "# this is a conversion so that and is the external one\n", "# C₁ = (P1 ∨ P3) ∧ (P1 ∨ P4); C₂ = (P2 ∨ P3) ∧ (P2 ∨ P4)\n", "# c1 and c2\n", "# \"\"\"\n", "# p1 = pl.col(a) < pl.col(c)\n", "# p2 = pl.col(c) < pl.col(b)\n", "# p3 = pl.col(a) < pl.col(d)\n", "# p4 = pl.col(d) < pl.col(b)\n", "# return p1 | p3, p1 | p4, p2 | p3, p2 | p4\n", "# thetime.join_where(updf, *interval_intersect('time', 'time2', 'start_time', 'success_time'))" ] }, { "cell_type": "code", "execution_count": 10, "id": "f0d5b469-4955-4ae2-944b-a397f635aeda", "metadata": {}, "outputs": [], "source": [ "def interval_intersect(a,b,c,d):\n", " \"\"\"\n", " a,b intersect c,d\n", " the input are column names\n", " \"\"\"\n", " p1 = pl.col(a) < pl.col(c)\n", " p2 = pl.col(c) < pl.col(b)\n", " p3 = pl.col(a) < pl.col(d)\n", " p4 = pl.col(d) < pl.col(b)\n", " return (p1 & p2) | (p3 & p4)\n", "t1 = thetime.join_where(updf, pl.col('time') < pl.col('start_time'), pl.col('start_time') < pl.col('time2'))\n", "t2 = thetime.join_where(updf, pl.col('time') < pl.col('success_time'), pl.col('success_time') < pl.col('time2'))\n", "final = pl.concat([t1,t2], how='vertical').unique()" ] }, { "cell_type": "code", "execution_count": 11, "id": "350942de-0c14-4c66-af8e-4ece3c82e68a", "metadata": {}, "outputs": [], "source": [ "# Usando start_time e non success_time-exectime stiamo contando il tempo causato da una task tra attesa+esecuzione.\n", "# Non e' proprio corretto ma e' \"simile\" al Load Average\n", "# 1 = occupiedtime = 1min\n", "loaddf = final.with_columns(slottasktime=pl.min_horizontal('success_time', 'time2')-pl.max_horizontal('start_time', 'time')).group_by('worker','time').agg(\n", " pl.sum('slottasktime').alias('occupiedtime')\n", ").with_columns(load=pl.col('occupiedtime')/timedelta(minutes=1))" ] }, { "cell_type": "code", "execution_count": 12, "id": "33a9fdf1-f0ac-46e0-a81d-bd4ab0eaed28", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import altair as alt\n", "alt.data_transformers.enable(\"vegafusion\")\n", "alt.Chart(loaddf).mark_line().encode(\n", " x='time:T',\n", " y='load',\n", " color='worker:N',\n", ").properties(\n", " width=1200,\n", ").interactive()" ] }, { "cell_type": "code", "execution_count": 13, "id": "a3c8fb02-10b1-40f6-8012-88bcc29e1c06", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (9, 5)
statisticworkertimeoccupiedtimeload
strstrstrstrf64
"count""30776""30776""30776"30776.0
"null_count""0""0""0"0.0
"mean"null"2024-12-16 11:17:57.042500+00:…"0:00:15.300420"0.255007
"std"nullnullnull0.300099
"min""10-1""2024-12-15 23:00:00+00:00""0:00:00.011000"0.000183
"25%"null"2024-12-16 05:44:00+00:00""0:00:01.602000"0.0267
"50%"null"2024-12-16 11:29:00+00:00""0:00:07.009000"0.116817
"75%"null"2024-12-16 17:01:00+00:00""0:00:26.260000"0.437667
"max""9-4""2024-12-16 22:59:00+00:00""0:06:41.711000"6.695183
" ], "text/plain": [ "shape: (9, 5)\n", "┌────────────┬────────┬─────────────────────────────────┬────────────────┬──────────┐\n", "│ statistic ┆ worker ┆ time ┆ occupiedtime ┆ load │\n", "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ str ┆ str ┆ str ┆ str ┆ f64 │\n", "╞════════════╪════════╪═════════════════════════════════╪════════════════╪══════════╡\n", "│ count ┆ 30776 ┆ 30776 ┆ 30776 ┆ 30776.0 │\n", "│ null_count ┆ 0 ┆ 0 ┆ 0 ┆ 0.0 │\n", "│ mean ┆ null ┆ 2024-12-16 11:17:57.042500+00:… ┆ 0:00:15.300420 ┆ 0.255007 │\n", "│ std ┆ null ┆ null ┆ null ┆ 0.300099 │\n", "│ min ┆ 10-1 ┆ 2024-12-15 23:00:00+00:00 ┆ 0:00:00.011000 ┆ 0.000183 │\n", "│ 25% ┆ null ┆ 2024-12-16 05:44:00+00:00 ┆ 0:00:01.602000 ┆ 0.0267 │\n", "│ 50% ┆ null ┆ 2024-12-16 11:29:00+00:00 ┆ 0:00:07.009000 ┆ 0.116817 │\n", "│ 75% ┆ null ┆ 2024-12-16 17:01:00+00:00 ┆ 0:00:26.260000 ┆ 0.437667 │\n", "│ max ┆ 9-4 ┆ 2024-12-16 22:59:00+00:00 ┆ 0:06:41.711000 ┆ 6.695183 │\n", "└────────────┴────────┴─────────────────────────────────┴────────────────┴──────────┘" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "loaddf.describe()" ] }, { "cell_type": "code", "execution_count": 14, "id": "10ef672b-0629-4e1f-9ba9-c94c0ab17a39", "metadata": {}, "outputs": [], "source": [ "oneloaddf = loaddf.group_by('time').agg(pl.col('load').mean()).sort('time')" ] }, { "cell_type": "code", "execution_count": 15, "id": "ac124411-cc12-4924-adea-781c55e25a0a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "alt.Chart(oneloaddf).mark_line().encode(\n", " x='time:T',\n", " y='load',\n", ").properties(\n", " width=1200,\n", ").interactive()" ] }, { "cell_type": "markdown", "id": "9454d46d-fd78-4cb3-81c6-f6b79b921cc9", "metadata": {}, "source": [ "## Cose ancora da fare:\n", "- Plottare non il Load ma il REAL \"doing something/nothing\" utilizzando il vero execution start time.\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "b9c29ddf-c888-438c-899e-c82c7a6cec30", "metadata": {}, "outputs": [], "source": [ "def interval_intersect(a,b,c,d):\n", " \"\"\"\n", " a,b intersect c,d\n", " the input are column names\n", " \"\"\"\n", " p1 = pl.col(a) < pl.col(c)\n", " p2 = pl.col(c) < pl.col(b)\n", " p3 = pl.col(a) < pl.col(d)\n", " p4 = pl.col(d) < pl.col(b)\n", " return (p1 & p2) | (p3 & p4)\n", "updf2 = updf.with_columns(execstart_time=pl.col('success_time') - pl.duration(microseconds=pl.col('exectime')*1e6))\n", "t21 = thetime.join_where(updf2, pl.col('time') < pl.col('execstart_time'), pl.col('execstart_time') < pl.col('time2'))\n", "t22 = thetime.join_where(updf2, pl.col('time') < pl.col('success_time'), pl.col('success_time') < pl.col('time2'))\n", "final2 = pl.concat([t21,t22], how='vertical').unique()" ] }, { "cell_type": "code", "execution_count": 17, "id": "5b48bd8f-e213-4757-bcec-6d1c61a593ce", "metadata": {}, "outputs": [], "source": [ "# Usando success_time-exectime stiamo contando il tempo in cui c'era una task in lavorazione\n", "# Non e' proprio corretto ma e' \"simile\" al Load Average\n", "# 1 = occupiedtime = 1min\n", "utildf = final2.with_columns(slottasktime=pl.min_horizontal('success_time', 'time2')-pl.max_horizontal('execstart_time', 'time')).group_by('worker','time').agg(\n", " pl.sum('slottasktime').alias('occupiedtime'),\n", ").with_columns(util=pl.col('occupiedtime')/timedelta(minutes=1)*100)" ] }, { "cell_type": "code", "execution_count": 18, "id": "47021f59-f926-4981-ac5a-db568e417095", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import altair as alt\n", "alt.data_transformers.enable(\"vegafusion\")\n", "alt.Chart(utildf).mark_line().encode(\n", " x='time:T',\n", " y='util',\n", " color='worker:N',\n", ").properties(\n", " width=1200,\n", ").interactive()" ] }, { "cell_type": "code", "execution_count": 19, "id": "c17c5fd4-6148-4bdc-a742-bdc2634e49a4", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (9, 5)
statisticworkertimeoccupiedtimeutil
strstrstrstrf64
"count""30776""30776""30776"30776.0
"null_count""0""0""0"0.0
"mean"null"2024-12-16 11:17:57.042500+00:…"0:00:11.713101"19.521836
"std"nullnullnull19.590662
"min""10-1""2024-12-15 23:00:00+00:00""0:00:00.010660"0.017767
"25%"null"2024-12-16 05:44:00+00:00""0:00:01.531180"2.551967
"50%"null"2024-12-16 11:29:00+00:00""0:00:06.562577"10.937628
"75%"null"2024-12-16 17:01:00+00:00""0:00:22.926698"38.211163
"max""9-4""2024-12-16 22:59:00+00:00""0:06:41.309068"668.848447
" ], "text/plain": [ "shape: (9, 5)\n", "┌────────────┬────────┬─────────────────────────────────┬────────────────┬────────────┐\n", "│ statistic ┆ worker ┆ time ┆ occupiedtime ┆ util │\n", "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ str ┆ str ┆ str ┆ str ┆ f64 │\n", "╞════════════╪════════╪═════════════════════════════════╪════════════════╪════════════╡\n", "│ count ┆ 30776 ┆ 30776 ┆ 30776 ┆ 30776.0 │\n", "│ null_count ┆ 0 ┆ 0 ┆ 0 ┆ 0.0 │\n", "│ mean ┆ null ┆ 2024-12-16 11:17:57.042500+00:… ┆ 0:00:11.713101 ┆ 19.521836 │\n", "│ std ┆ null ┆ null ┆ null ┆ 19.590662 │\n", "│ min ┆ 10-1 ┆ 2024-12-15 23:00:00+00:00 ┆ 0:00:00.010660 ┆ 0.017767 │\n", "│ 25% ┆ null ┆ 2024-12-16 05:44:00+00:00 ┆ 0:00:01.531180 ┆ 2.551967 │\n", "│ 50% ┆ null ┆ 2024-12-16 11:29:00+00:00 ┆ 0:00:06.562577 ┆ 10.937628 │\n", "│ 75% ┆ null ┆ 2024-12-16 17:01:00+00:00 ┆ 0:00:22.926698 ┆ 38.211163 │\n", "│ max ┆ 9-4 ┆ 2024-12-16 22:59:00+00:00 ┆ 0:06:41.309068 ┆ 668.848447 │\n", "└────────────┴────────┴─────────────────────────────────┴────────────────┴────────────┘" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "utildf.describe()" ] }, { "cell_type": "code", "execution_count": 20, "id": "0595e945-56b6-4959-aae4-242f3062d80e", "metadata": {}, "outputs": [], "source": [ "oneutildf = utildf.group_by('time').agg(pl.col('util').mean()).sort('time')" ] }, { "cell_type": "code", "execution_count": 21, "id": "043e580f-3fb2-4c57-906e-1a0ea99d483c", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "alt.Chart(oneutildf).mark_line().encode(\n", " x='time:T',\n", " y='util',\n", ").properties(\n", " width=1200,\n", ").interactive()" ] }, { "cell_type": "code", "execution_count": 22, "id": "1241fac3-b4d1-48cb-b3b4-f088d3729ac4", "metadata": {}, "outputs": [], "source": [ "# re_restapi.libs.advenir.advenir_send_charge do a lot of retry and this inflate artificially his waittime\n", "dtimedf = updf.drop_nulls().filter(pl.col('taskname') != 're_restapi.libs.advenir.advenir_send_charge').with_columns(execstarttime=pl.col('success_time')-pl.duration(microseconds=pl.col('exectime')*1e6)).with_columns(waittime=pl.col('execstarttime')-pl.col('start_time'))" ] }, { "cell_type": "code", "execution_count": 28, "id": "6fab0616-3625-4881-a06e-fdca5a5a2b00", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (2_883_227, 8)
taskidstart_timesuccess_timeexectimetasknameworkerexecstarttimewaittime
strdatetime[μs, UTC]datetime[μs, UTC]f64strstrdatetime[μs, UTC]duration[μs]
"84228d03-3f4b-4b6b-8ad6-96dcea…2024-12-16 09:04:11.101 UTC2024-12-16 09:04:20.720 UTC0.96336"re_restapi.libs.metrics.task_p…"11-4"2024-12-16 09:04:19.756640 UTC8s 655640µs
"761cd466-ce67-47e8-94cb-e756e1…2024-12-16 14:40:09.237 UTC2024-12-16 14:40:19.229 UTC1.33955"re_restapi.libs.postpower.task…"10-2"2024-12-16 14:40:17.889450 UTC8s 652450µs
"bbdb96ac-556b-49ca-b97a-a5fe37…2024-12-16 09:04:11.631 UTC2024-12-16 09:04:20.741 UTC0.52521"re_restapi.libs.ipsum.task_pos…"11-3"2024-12-16 09:04:20.215790 UTC8s 584790µs
"af30aa19-b6a5-44d5-ad9e-99bf64…2024-12-16 09:04:11.355 UTC2024-12-16 09:04:20.834 UTC0.93162"re_restapi.libs.postpower.task…"11-1"2024-12-16 09:04:19.902380 UTC8s 547380µs
"d1cc57b5-9ed8-424b-ae47-072be6…2024-12-16 08:45:12.467 UTC2024-12-16 08:45:21.471 UTC0.51711"re_restapi.libs.postpower.task…"7-4"2024-12-16 08:45:20.953891 UTC8s 486891µs
……………………
"9ea103c5-9aec-4dbb-b7dc-610be6…2024-12-16 01:54:34.456 UTC2024-12-16 01:54:34.535 UTC0.07925"re_restapi.libs.ipsum.send_sin…"6-3"2024-12-16 01:54:34.455750 UTC-250µs
"418b67d6-12b5-49d8-84e4-b726d0…2024-12-16 05:01:30.723 UTC2024-12-16 05:01:30.798 UTC0.07526"re_restapi.libs.ipsum.send_sin…"9-3"2024-12-16 05:01:30.722740 UTC-260µs
"0b03c6a5-8478-4561-b01d-1ad2d8…2024-12-16 03:15:48.990 UTC2024-12-16 03:15:49.070 UTC0.08026"re_restapi.libs.ipsum.send_sin…"8-2"2024-12-16 03:15:48.989740 UTC-260µs
"e6fdf1f0-28de-4e0d-9801-4f8c0b…2024-12-16 03:27:42.984 UTC2024-12-16 03:27:43.065 UTC0.08126"re_restapi.libs.ipsum.send_sin…"7-3"2024-12-16 03:27:42.983740 UTC-260µs
"c6714c12-98ab-4f43-8d09-92fb4b…2024-12-16 03:37:24.301 UTC2024-12-16 03:37:24.381 UTC0.08029"re_restapi.libs.ipsum.send_sin…"8-2"2024-12-16 03:37:24.300710 UTC-290µs
" ], "text/plain": [ "shape: (2_883_227, 8)\n", "┌─────────────┬────────────┬────────────┬──────────┬────────────┬────────┬────────────┬────────────┐\n", "│ taskid ┆ start_time ┆ success_ti ┆ exectime ┆ taskname ┆ worker ┆ execstartt ┆ waittime │\n", "│ --- ┆ --- ┆ me ┆ --- ┆ --- ┆ --- ┆ ime ┆ --- │\n", "│ str ┆ datetime[μ ┆ --- ┆ f64 ┆ str ┆ str ┆ --- ┆ duration[μ │\n", "│ ┆ s, UTC] ┆ datetime[μ ┆ ┆ ┆ ┆ datetime[μ ┆ s] │\n", "│ ┆ ┆ s, UTC] ┆ ┆ ┆ ┆ s, UTC] ┆ │\n", "╞═════════════╪════════════╪════════════╪══════════╪════════════╪════════╪════════════╪════════════╡\n", "│ 84228d03-3f ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.96336 ┆ re_restapi ┆ 11-4 ┆ 2024-12-16 ┆ 8s │\n", "│ 4b-4b6b-8ad ┆ 09:04:11.1 ┆ 09:04:20.7 ┆ ┆ .libs.metr ┆ ┆ 09:04:19.7 ┆ 655640µs │\n", "│ 6-96dcea… ┆ 01 UTC ┆ 20 UTC ┆ ┆ ics.task_p ┆ ┆ 56640 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ 761cd466-ce ┆ 2024-12-16 ┆ 2024-12-16 ┆ 1.33955 ┆ re_restapi ┆ 10-2 ┆ 2024-12-16 ┆ 8s │\n", "│ 67-47e8-94c ┆ 14:40:09.2 ┆ 14:40:19.2 ┆ ┆ .libs.post ┆ ┆ 14:40:17.8 ┆ 652450µs │\n", "│ b-e756e1… ┆ 37 UTC ┆ 29 UTC ┆ ┆ power.task ┆ ┆ 89450 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ bbdb96ac-55 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.52521 ┆ re_restapi ┆ 11-3 ┆ 2024-12-16 ┆ 8s │\n", "│ 6b-49ca-b97 ┆ 09:04:11.6 ┆ 09:04:20.7 ┆ ┆ .libs.ipsu ┆ ┆ 09:04:20.2 ┆ 584790µs │\n", "│ a-a5fe37… ┆ 31 UTC ┆ 41 UTC ┆ ┆ m.task_pos ┆ ┆ 15790 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ af30aa19-b6 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.93162 ┆ re_restapi ┆ 11-1 ┆ 2024-12-16 ┆ 8s │\n", "│ a5-44d5-ad9 ┆ 09:04:11.3 ┆ 09:04:20.8 ┆ ┆ .libs.post ┆ ┆ 09:04:19.9 ┆ 547380µs │\n", "│ e-99bf64… ┆ 55 UTC ┆ 34 UTC ┆ ┆ power.task ┆ ┆ 02380 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ d1cc57b5-9e ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.51711 ┆ re_restapi ┆ 7-4 ┆ 2024-12-16 ┆ 8s │\n", "│ d8-424b-ae4 ┆ 08:45:12.4 ┆ 08:45:21.4 ┆ ┆ .libs.post ┆ ┆ 08:45:20.9 ┆ 486891µs │\n", "│ 7-072be6… ┆ 67 UTC ┆ 71 UTC ┆ ┆ power.task ┆ ┆ 53891 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", "│ 9ea103c5-9a ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.07925 ┆ re_restapi ┆ 6-3 ┆ 2024-12-16 ┆ -250µs │\n", "│ ec-4dbb-b7d ┆ 01:54:34.4 ┆ 01:54:34.5 ┆ ┆ .libs.ipsu ┆ ┆ 01:54:34.4 ┆ │\n", "│ c-610be6… ┆ 56 UTC ┆ 35 UTC ┆ ┆ m.send_sin ┆ ┆ 55750 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ 418b67d6-12 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.07526 ┆ re_restapi ┆ 9-3 ┆ 2024-12-16 ┆ -260µs │\n", "│ b5-49d8-84e ┆ 05:01:30.7 ┆ 05:01:30.7 ┆ ┆ .libs.ipsu ┆ ┆ 05:01:30.7 ┆ │\n", "│ 4-b726d0… ┆ 23 UTC ┆ 98 UTC ┆ ┆ m.send_sin ┆ ┆ 22740 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ 0b03c6a5-84 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.08026 ┆ re_restapi ┆ 8-2 ┆ 2024-12-16 ┆ -260µs │\n", "│ 78-4561-b01 ┆ 03:15:48.9 ┆ 03:15:49.0 ┆ ┆ .libs.ipsu ┆ ┆ 03:15:48.9 ┆ │\n", "│ d-1ad2d8… ┆ 90 UTC ┆ 70 UTC ┆ ┆ m.send_sin ┆ ┆ 89740 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ e6fdf1f0-28 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.08126 ┆ re_restapi ┆ 7-3 ┆ 2024-12-16 ┆ -260µs │\n", "│ de-4e0d-980 ┆ 03:27:42.9 ┆ 03:27:43.0 ┆ ┆ .libs.ipsu ┆ ┆ 03:27:42.9 ┆ │\n", "│ 1-4f8c0b… ┆ 84 UTC ┆ 65 UTC ┆ ┆ m.send_sin ┆ ┆ 83740 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "│ c6714c12-98 ┆ 2024-12-16 ┆ 2024-12-16 ┆ 0.08029 ┆ re_restapi ┆ 8-2 ┆ 2024-12-16 ┆ -290µs │\n", "│ ab-4f43-8d0 ┆ 03:37:24.3 ┆ 03:37:24.3 ┆ ┆ .libs.ipsu ┆ ┆ 03:37:24.3 ┆ │\n", "│ 9-92fb4b… ┆ 01 UTC ┆ 81 UTC ┆ ┆ m.send_sin ┆ ┆ 00710 UTC ┆ │\n", "│ ┆ ┆ ┆ ┆ … ┆ ┆ ┆ │\n", "└─────────────┴────────────┴────────────┴──────────┴────────────┴────────┴────────────┴────────────┘" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# waittime negative is OK because the start_time is the time of writing of the log that takes some delay in the 1ms order of magnitude.\n", "dtimedf.sort('waittime').reverse()" ] }, { "cell_type": "code", "execution_count": 29, "id": "97c83065-e462-429e-8417-12d47c0fc210", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "alt.Chart(dtimedf).mark_bar().encode(\n", " alt.X(\"waittime:Q\").bin(maxbins=100),\n", " alt.Y('count()').scale(type=\"log\"),\n", " alt.Tooltip('count()')\n", ").properties(width=1200).interactive()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }