Skip to content

daily_validation

measurement.dash_apps.daily_validation ¤

Attributes¤

DATA_GRANULAR: pd.DataFrame = pd.DataFrame() module-attribute ¤

DATA_SUMMARY: pd.DataFrame = pd.DataFrame() module-attribute ¤

SELECTED_DAY: date | None = None module-attribute ¤

app = DjangoDash('DailyValidation', external_stylesheets=[dbc.themes.BOOTSTRAP, '/static/styles/dashstyle.css']) module-attribute ¤

detail_date_picker = html.Div(children=[html.Div(children=['Open detailed view'], style={'display': 'inline-block', 'padding-right': '5px'}), dcc.DatePickerSingle(id='detail-date-picker', display_format='YYYY-MM-DD', min_date_allowed=None, max_date_allowed=None)], style={'display': 'inline-block', 'width': '50%', 'text-align': 'right'}) module-attribute ¤

filters = html.Div(children=[filters_row1, filters_row2]) module-attribute ¤

filters_row1 = html.Div(children=[html.Div([html.Label('Station:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='station_drop', options=[], value=None)], style={'margin-right': '20px', 'width': '286px', 'display': 'inline-block'}), html.Div([html.Label('Variable:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='variable_drop', options=[], value=None)], style={'margin-right': '20px', 'width': '286px', 'display': 'inline-block'}), html.Div([html.Label('Date Range:', style={'display': 'block', 'font-weight': 'bold'}), dcc.DatePickerRange(id='date_range_picker', display_format='YYYY-MM-DD', start_date=None, end_date=None)], style={'width': '286px', 'display': 'inline-block'})], style={'display': 'flex', 'justify-content': 'flex-start', 'margin-bottom': '10px'}) module-attribute ¤

filters_row2 = html.Div(children=[html.Div([html.Label('Minimum:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Input(id='minimum_input', type='number', value=None)], style={'margin-right': '20px', 'width': '286px'}), html.Div([html.Label('Maximum:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Input(id='maximum_input', type='number', value=None)], style={'margin-right': '20px', 'width': '286px'}), html.Div([html.Label('Validation status:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='validation_status_drop', options=[{'label': 'Validated', 'value': 'validated'}, {'label': 'Not validated', 'value': 'not_validated'}], value='not_validated')], style={'width': '286px'})], style={'display': 'flex', 'justify-content': 'flex-start'}) module-attribute ¤

menu = html.Div(children=[html.Div(children=[html.Button('Validate', id='save-button')], style={'display': 'inline-block', 'width': '50%'}), detail_date_picker], style={'background-color': '#f0f0f0', 'width': '100%'}) module-attribute ¤

plot = html.Div(children=[dcc.Graph(id='plot', figure=create_empty_plot(), style={'width': '100%'}), dcc.RadioItems(id='plot_radio', options=[{'value': c, 'label': c.capitalize()} for c in ['value', 'maximum', 'minimum']], value='value', style={'width': '100px'}, labelStyle={'display': 'block'})], style={'display': 'flex', 'justify-content': 'space-between', 'height': '400px'}) module-attribute ¤

status_message = html.Div(id='status-message', children=[''], style={'min-height': '20px', 'padding-top': '5px', 'padding-bottom': '10px'}) module-attribute ¤

table_daily = AgGrid(id='table_daily', rowData=[], columnDefs=create_columns_daily(), columnSize='sizeToFit', defaultColDef={'resizable': True, 'sortable': True, 'checkboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelectionFilteredOnly': True}, dashGridOptions={'rowSelection': 'multiple', 'suppressRowClickSelection': True}, selectAll=True, getRowId='params.data.date') module-attribute ¤

table_detail = AgGrid(id='table_detail', rowData=[], columnDefs=create_columns_detail(), columnSize='sizeToFit', defaultColDef={'resizable': True, 'sortable': True, 'checkboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelectionFilteredOnly': True}, dashGridOptions={'rowSelection': 'multiple', 'suppressRowClickSelection': True}, selectAll=True, getRowId='params.data.id') module-attribute ¤

Classes¤

Variable ¤

Bases: PermissionsBase

A variable with a physical meaning.

Such as precipitation, wind speed, wind direction, soil moisture, including the associated unit. It also includes metadata to help identify what is a reasonable value for the data, to flag outliers and to help with the validation process.

The nature of the variable can be one of the following:

  • sum: Cumulative value over a period of time.
  • average: Average value over a period of time.
  • value: One-off value.

Attributes:

Name Type Description
variable_id AutoField

Primary key.

variable_code CharField

Code of the variable, eg. airtemperature.

name CharField

Human-readable name of the variable, eg. Air temperature.

unit ForeignKey

Unit of the variable.

maximum DecimalField

Maximum value allowed for the variable.

minimum DecimalField

Minimum value allowed for the variable.

diff_error DecimalField

If two sequential values in the time-series data of this variable differ by more than this value, the validation process can mark this with an error flag.

outlier_limit DecimalField

The statistical deviation for defining outliers, in times the standard deviation (sigma).

null_limit DecimalField

The max % of null values (missing, caused by e.g. equipment malfunction) allowed for hourly, daily, monthly data. Cumulative values are not deemed trustworthy if the number of missing values in a given period is greater than the null_limit.

nature CharField

Nature of the variable, eg. if it represents a one-off value, the average over a period of time or the cumulative value over a period

Attributes¤
is_cumulative: bool property ¤

Return True if the nature of the variable is sum.

Functions¤
__str__() ¤

Return the string representation of the object.

Source code in variable\models.py
165
166
167
def __str__(self) -> str:
    """Return the string representation of the object."""
    return str(self.name)
clean() ¤

Validate the model fields.

Source code in variable\models.py
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def clean(self) -> None:
    """Validate the model fields."""
    if self.maximum < self.minimum:
        raise ValidationError(
            {
                "maximum": "The maximum value must be greater than the minimum "
                "value."
            }
        )
    if not self.variable_code.isidentifier():
        raise ValidationError(
            {
                "variable_code": "The variable code must be a valid Python "
                "identifier. Only letters, numbers and underscores are allowed, and"
                " it cannot start with a number."
            }
        )
    return super().clean()
get_absolute_url() ¤

Get the absolute URL of the object.

Source code in variable\models.py
169
170
171
def get_absolute_url(self) -> str:
    """Get the absolute URL of the object."""
    return reverse("variable:variable_detail", kwargs={"pk": self.pk})

Functions¤

callbacks(in_submit_clicks, in_save_clicks, in_detail_date, in_plot_radio_value, in_tabs_value, in_station, in_variable, in_start_date, in_end_date, in_minimum, in_maximum, in_daily_selected_rows, in_daily_row_data, in_detail_selected_rows, in_detail_row_data, in_validation_status) ¤

Callbacks for daily validation app

Parameters:

Name Type Description Default
in_submit_clicks int

Number of times submit-button was clicked

required
in_save_clicks int

Number of times save-button was clicked

required
in_detail_date str

Date for detail view

required
in_plot_radio_value str

Value of plot radio button

required
in_tabs_value str

Value of tabs

required
in_station str

Station from filters

required
in_variable str

Variable from filters

required
in_start_date str

Start date from filters

required
in_end_date str

End date from filters

required
in_minimum float

Minimum from filters

required
in_maximum float

Maximum from filters

required
in_daily_selected_rows list[dict]

Selected rows in table_daily

required
in_daily_row_data list[dict]

Full row data for table_daily

required
in_detail_selected_rows list[dict]

Selected rows in table_detail

required
in_detail_row_data list[dict]

Full row data for table_detail

required

Returns:

Name Type Description
out_loading_top no_update

Loading spinner for top

out_loading no_update

Loading spinner for bottom

out_status str

Status message

out_plot Figure

Plot

out_daily_row_data list[dict]

Data for daily table

out_detail_row_data list[dict]

Data for detail table

out_daily_selected_rows list[dict]

Selected rows in daily table

out_detail_selected_rows list[dict]

Selected rows in detail table

out_tab_detail_disabled bool

Disabled status for detail tab

out_tab_detail_label str

Label for detail tab

out_tabs_value str

Value for tabs

out_save_label str

Label for save button

Source code in measurement\dash_apps\daily_validation.py
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
@app.callback(
    [
        Output("loading_top", "children"),
        Output("loading", "children"),
        Output("status-message", "children"),
        Output("plot", "figure"),
        Output("table_daily", "rowData"),
        Output("table_detail", "rowData"),
        Output("table_daily", "selectedRows"),
        Output("table_detail", "selectedRows"),
        Output("tab-detail", "disabled"),
        Output("tab-detail", "label"),
        Output("tabs", "value"),
        Output("save-button", "children"),
    ],
    [
        Input("submit-button", "n_clicks"),
        Input("save-button", "n_clicks"),
        Input("detail-date-picker", "date"),
        Input("plot_radio", "value"),
    ],
    [
        State("tabs", "value"),
        State("station_drop", "value"),
        State("variable_drop", "value"),
        State("date_range_picker", "start_date"),
        State("date_range_picker", "end_date"),
        State("minimum_input", "value"),
        State("maximum_input", "value"),
        State("table_daily", "selectedRows"),
        State("table_daily", "rowData"),
        State("table_detail", "selectedRows"),
        State("table_detail", "rowData"),
        State("validation_status_drop", "value"),
    ],
    prevent_initial_call=True,
)
def callbacks(
    in_submit_clicks: int,
    in_save_clicks: int,
    in_detail_date: str,
    in_plot_radio_value: str,
    in_tabs_value: str,
    in_station: str,
    in_variable: str,
    in_start_date: str,
    in_end_date: str,
    in_minimum: float,
    in_maximum: float,
    in_daily_selected_rows: list[dict],
    in_daily_row_data: list[dict],
    in_detail_selected_rows: list[dict],
    in_detail_row_data: list[dict],
    in_validation_status: str,
) -> tuple[
    dash.no_update,
    dash.no_update,
    str,
    go.Figure,
    list[dict],
    list[dict],
    list[dict],
    list[dict],
    bool,
    str,
    str,
    str,
]:
    """Callbacks for daily validation app

    Args:
        in_submit_clicks (int): Number of times submit-button was clicked
        in_save_clicks (int): Number of times save-button was clicked
        in_detail_date (str): Date for detail view
        in_plot_radio_value (str): Value of plot radio button
        in_tabs_value (str): Value of tabs
        in_station (str): Station from filters
        in_variable (str): Variable from filters
        in_start_date (str): Start date from filters
        in_end_date (str): End date from filters
        in_minimum (float): Minimum from filters
        in_maximum (float): Maximum from filters
        in_daily_selected_rows (list[dict]): Selected rows in table_daily
        in_daily_row_data (list[dict]): Full row data for table_daily
        in_detail_selected_rows (list[dict]): Selected rows in table_detail
        in_detail_row_data (list[dict]): Full row data for table_detail

    Returns:
        out_loading_top (dash.no_update): Loading spinner for top
        out_loading (dash.no_update): Loading spinner for bottom
        out_status (str): Status message
        out_plot (go.Figure): Plot
        out_daily_row_data (list[dict]): Data for daily table
        out_detail_row_data (list[dict]): Data for detail table
        out_daily_selected_rows (list[dict]): Selected rows in daily table
        out_detail_selected_rows (list[dict]): Selected rows in detail table
        out_tab_detail_disabled (bool): Disabled status for detail tab
        out_tab_detail_label (str): Label for detail tab
        out_tabs_value (str): Value for tabs
        out_save_label (str): Label for save button
    """
    global SELECTED_DAY, DATA_SUMMARY, DATA_GRANULAR

    ctx = dash.callback_context
    input_id = ctx.triggered[0]["prop_id"].split(".")[0]

    out_loading_top = dash.no_update
    out_loading = dash.no_update
    out_status = dash.no_update
    out_plot = dash.no_update
    out_daily_row_data = dash.no_update
    out_detail_row_data = dash.no_update
    out_daily_selected_rows = dash.no_update
    out_detail_selected_rows = dash.no_update
    out_tab_detail_disabled = dash.no_update
    out_tab_detail_label = dash.no_update
    out_tabs_value = dash.no_update
    out_save_label = dash.no_update

    data_refresh_required = False
    daily_table_refresh_required = False
    detail_table_refresh_required = False
    daily_table_reset_selection = False
    detail_table_reset_selection = False
    plot_refresh_required = False

    # Button: Submit
    if input_id == "submit-button":
        out_status = ""
        out_save_label = (
            "Validate"
            if in_validation_status == "not_validated"
            else "Reset Validation"
        )
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        daily_table_reset_selection = True
        detail_table_reset_selection = True
        plot_refresh_required = True

    # Button: Save (daily)
    if (
        input_id == "save-button"
        and in_tabs_value == "tab-daily"
        and in_validation_status == "not_validated"
    ):
        selected_dates = {row["date"] for row in in_daily_selected_rows}
        data_to_validate = [
            {
                "date": row["date"].split("T")[0],
                "validate?": True,
                "deactivate?": row["date"] not in selected_dates,
                "station": in_station,
                "variable": in_variable,
            }
            for row in in_daily_row_data
        ]
        save_validated_days(pd.DataFrame.from_records(data_to_validate))
        out_status = "Validation successful"
        data_refresh_required = True
        daily_table_refresh_required = True
        plot_refresh_required = True

    # Button: Save (detail)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-detail"
        and in_validation_status == "not_validated"
    ):
        selected_ids = {row["id"] for row in in_detail_selected_rows}
        data_to_validate = [
            {
                "id": row["id"],
                "validate?": True,
                "deactivate?": row["id"] not in selected_ids,
                "value": row["value"],
                "minimum": row["minimum"],
                "maximum": row["maximum"],
            }
            for row in in_detail_row_data
        ]
        save_validated_entries(pd.DataFrame.from_records(data_to_validate))
        out_status = "Validation successful"
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        plot_refresh_required = True

    # Button: Reset (daily)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-daily"
        and in_validation_status == "validated"
    ):
        reset_validated_days(
            variable=in_variable,
            station=in_station,
            start_date=in_start_date,
            end_date=in_end_date,
        )
        out_status = "Validation reset"
        data_refresh_required = True
        daily_table_refresh_required = True
        daily_table_reset_selection = True
        plot_refresh_required = True

    # Button: Reset (detail)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-detail"
        and in_validation_status == "validated"
    ):
        reset_validated_entries(ids=[row["id"] for row in in_detail_row_data])
        out_status = "Validation reset"
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        detail_table_reset_selection = True
        plot_refresh_required = True

    # Date picker
    elif input_id == "detail-date-picker":
        new_selected_day = next(
            (
                d.date()
                for d in DATA_SUMMARY["date"]
                if d.strftime("%Y-%m-%d") == in_detail_date
            ),
            None,
        )
        if new_selected_day is not None:
            detail_table_refresh_required = True
            detail_table_reset_selection = True
            out_tab_detail_disabled = False
            out_tab_detail_label = (
                f"Detail of Selected Day ({new_selected_day.strftime('%Y-%m-%d')})"
            )
            out_tabs_value = "tab-detail"
            out_status = ""
            SELECTED_DAY = new_selected_day
        else:
            out_status = "No data for selected day"

    # Plot radio
    elif input_id == "plot_radio":
        plot_refresh_required = True

    # Reload data
    if data_refresh_required:
        DATA_SUMMARY, DATA_GRANULAR = generate_validation_report(
            station=in_station,
            variable=in_variable,
            start_time=in_start_date,
            end_time=in_end_date,
            minimum=Decimal(in_minimum) if in_minimum is not None else None,
            maximum=Decimal(in_maximum) if in_maximum is not None else None,
            is_validated=in_validation_status == "validated",
        )

    # Refresh plot
    if plot_refresh_required:
        if not DATA_GRANULAR.empty:
            out_plot = create_validation_plot(
                data=DATA_GRANULAR,
                variable_name=Variable.objects.get(variable_code=in_variable).name,
                field=in_plot_radio_value,
            )
        else:
            out_plot = create_empty_plot()

    # Refresh daily table
    if daily_table_refresh_required:
        out_daily_row_data = DATA_SUMMARY.to_dict("records")

        # Reset daily table selection
        if daily_table_reset_selection:
            out_daily_selected_rows = out_daily_row_data

    # Refresh detail table
    if detail_table_refresh_required:
        if DATA_GRANULAR.empty:
            out_detail_row_data = []
        else:
            out_detail_row_data = DATA_GRANULAR[
                DATA_GRANULAR.time.dt.date == SELECTED_DAY
            ].to_dict("records")

        # Reset detail table selection
        if detail_table_reset_selection:
            out_detail_selected_rows = [
                row for row in out_detail_row_data if row["is_active"]
            ]

    return (
        out_loading_top,
        out_loading,
        out_status,
        out_plot,
        out_daily_row_data,
        out_detail_row_data,
        out_daily_selected_rows,
        out_detail_selected_rows,
        out_tab_detail_disabled,
        out_tab_detail_label,
        out_tabs_value,
        out_save_label,
    )

create_columns_daily() ¤

Creates columns for Daily Report table

Parameters:

Name Type Description Default
value_columns list

List of value columns

required

Returns:

Name Type Description
list list

List of columns

Source code in measurement\dash_apps\tables.py
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def create_columns_daily() -> list:
    """Creates columns for Daily Report table

    Args:
        value_columns (list): List of value columns

    Returns:
        list: List of columns
    """
    styles = create_style_conditions()

    columns = [
        {
            "valueGetter": {
                "function": "d3.timeParse('%Y-%m-%d')(params.data.date.split('T')[0])"
            },
            "headerName": "Date",
            "filter": "agDateColumnFilter",
            "valueFormatter": {"function": "params.data.date.split('T')[0]"},
            "sort": "asc",
            **styles["date"],
        },
        *[
            {
                "field": c,
                "headerName": c.capitalize(),
                "filter": "agNumberColumnFilter",
                "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
                **styles[c],
            }
            for c in ["value", "minimum", "maximum"]
        ],
        {
            "field": "daily_count_fraction",
            "headerName": "Daily count fraction",
            "filter": "agNumberColumnFilter",
            "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
            **styles["daily_count_fraction"],
        },
        {
            "field": "total_suspicious_entries",
            "headerName": "Suspicious entries",
            "filter": "agNumberColumnFilter",
            **styles["total_suspicious_entries"],
        },
    ]
    return columns

create_columns_detail() ¤

Creates columns for Detail table

Parameters:

Name Type Description Default
value_columns list

List of value columns

required

Returns:

Name Type Description
list list

List of columns

Source code in measurement\dash_apps\tables.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def create_columns_detail() -> list:
    """Creates columns for Detail table

    Args:
        value_columns (list): List of value columns

    Returns:
        list: List of columns
    """
    styles = create_style_conditions()

    columns = [
        {
            "field": "id",
            "headerName": "Measurement ID",
            "filter": "agNumberColumnFilter",
        },
        {
            "field": "time",
            "valueFormatter": {"function": "params.value.split('T')[1].split('+')[0]"},
            "headerName": "Time",
            "editable": True,
            "sort": "asc",
            **styles["time"],
        },
        *[
            {
                "field": c,
                "headerName": c.capitalize(),
                "filter": "agNumberColumnFilter",
                "editable": True,
                "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
                **styles[c],
            }
            for c in ["value", "minimum", "maximum"]
        ],
    ]
    return columns

create_empty_plot() ¤

Creates empty plot

Returns:

Type Description
scatter

px.Scatter: Plot

Source code in measurement\dash_apps\plots.py
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
def create_empty_plot() -> px.scatter:
    """Creates empty plot

    Returns:
        px.Scatter: Plot
    """
    fig = px.scatter(title="No data to plot")
    fig.update_layout(
        autosize=True,
        margin=dict(
            l=50,
            r=20,
            b=0,
            t=50,
        ),
        title_font=dict(
            size=14,
        ),
    )
    return fig

create_validation_plot(data, variable_name, field) ¤

Creates plot for Validation app

Parameters:

Name Type Description Default
data DataFrame

Data

required
variable_name str

Variable name

required
field str

'value', 'minimum' or 'maximum'

required

Returns:

Type Description
Figure

go.Figure: Plot

Source code in measurement\dash_apps\plots.py
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def create_validation_plot(
    data: pd.DataFrame, variable_name: str, field: str
) -> go.Figure:
    """Creates plot for Validation app

    Args:
        data (pd.DataFrame): Data
        variable_name (str): Variable name
        field (str): 'value', 'minimum' or 'maximum'

    Returns:
        go.Figure: Plot
    """

    def status(row):
        if not row["is_validated"]:
            return "Not validated"
        if row["is_active"]:
            return "Active"
        return "Inactive"

    color_map = {
        "Active": "#00CC96",
        "Inactive": "#636EFA",
        "Not validated": "black",
    }

    fig = px.scatter(
        data,
        x="time",
        y=field,
        color=data.apply(status, axis=1),
        color_discrete_map=color_map,
        labels={"time": "Date", field: f"{variable_name} ({field.capitalize()})"},
    )

    fig.update_traces(marker=dict(size=3))
    fig.update_layout(
        legend=dict(
            title=dict(text="Status", font=dict(size=12)),
            x=1,
            y=1,
            xanchor="auto",
            yanchor="auto",
        ),
        autosize=True,
        margin=dict(
            l=50,
            r=20,
            b=0,
            t=50,
        ),
    )

    return fig

generate_validation_report(station, variable, start_time, end_time, maximum, minimum, is_validated=False) ¤

Generates a report of the data.

Parameters:

Name Type Description Default
station str

Station of interest.

required
variable str

Variable of interest.

required
start_time str

Start time.

required
end_time str

End time.

required
maximum Decimal

The maximum allowed value.

required
minimum Decimal

The minimum allowed value.

required
is_validated bool

Whether to retrieve validated or non-validated data.

False

Returns:

Type Description
tuple[DataFrame, DataFrame]

A tuple with the summary report and the granular report.

Source code in measurement\validation.py
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def generate_validation_report(
    station: str,
    variable: str,
    start_time: str,
    end_time: str,
    maximum: Decimal,
    minimum: Decimal,
    is_validated: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """Generates a report of the data.

    Args:
        station: Station of interest.
        variable: Variable of interest.
        start_time: Start time.
        end_time: End time.
        maximum: The maximum allowed value.
        minimum: The minimum allowed value.
        is_validated: Whether to retrieve validated or non-validated data.

    Returns:
        A tuple with the summary report and the granular report.
    """
    var = Variable.objects.get(variable_code=variable)

    data = get_data_to_validate(station, variable, start_time, end_time, is_validated)
    if data.empty:
        return pd.DataFrame(), pd.DataFrame()

    suspicious = flag_suspicious_data(data, maximum, minimum, var.diff_error)
    summary = generate_daily_summary(
        data, suspicious, var.null_limit, var.is_cumulative
    )
    granular = pd.concat([data, suspicious], axis=1)
    return summary, granular

get_date_range(station, variable) ¤

Get the date range covered by a chosen station and variable.

Parameters:

Name Type Description Default
station str

Code for the chosen station

required
variable str

Code for the chosen variable

required

Returns:

Type Description
tuple[str, str]

tuple[str, str]: Start date, end date

Source code in measurement\filters.py
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def get_date_range(station: str, variable: str) -> tuple[str, str]:
    """Get the date range covered by a chosen station and variable.

    Args:
        station (str): Code for the chosen station
        variable (str): Code for the chosen variable

    Returns:
        tuple[str, str]: Start date, end date
    """
    filter_vals = Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
    ).aggregate(
        first_date=Min("time"),
        last_date=Max("time"),
    )

    first_date = to_local_time(filter_vals["first_date"]).strftime("%Y-%m-%d")
    last_date = to_local_time(filter_vals["last_date"]).strftime("%Y-%m-%d")
    return first_date, last_date

get_min_max(station, variable) ¤

Get the min and max of the data for a chosen station and variable.

Parameters:

Name Type Description Default
station str

Code for the chosen station

required
variable str

Code for the chosen variable

required

Returns:

Type Description
tuple[Decimal, Decimal]

tuple[Decimal, Decimal]: Min value, max value

Source code in measurement\filters.py
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def get_min_max(
    station, variable
) -> tuple[
    Decimal,
    Decimal,
]:
    """Get the min and max of the data for a chosen station and variable.

    Args:
        station (str): Code for the chosen station
        variable (str): Code for the chosen variable

    Returns:
        tuple[Decimal, Decimal]: Min value, max value
    """
    filter_vals = Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
    ).aggregate(
        min_value=Min("minimum"),
        max_value=Max("maximum"),
    )

    min_value = filter_vals["min_value"]
    max_value = filter_vals["max_value"]

    return min_value, max_value

get_station_options(station_codes) ¤

Get valid station options and default value based on permissions and data availability.

Parameters:

Name Type Description Default
station_codes list[str]

List of station codes based on permissions

required

Returns:

Type Description
tuple[list[dict[str, str]], str | None]

tuple[list[dict], str]: Options for the station dropdown, default value

Source code in measurement\filters.py
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
def get_station_options(
    station_codes: list[str],
) -> tuple[list[dict[str, str]], str | None]:
    """Get valid station options and default value based on permissions and data
    availability.

    Args:
        station_codes (list[str]): List of station codes based on permissions

    Returns:
        tuple[list[dict], str]: Options for the station dropdown, default value
    """
    stations_with_measurements = Station.objects.filter(
        ~Q(variables=""), station_code__in=station_codes
    ).values_list("station_code", flat=True)

    station_options = [
        {"label": station_code, "value": station_code}
        for station_code in stations_with_measurements
    ]
    station_value = station_options[0]["value"] if station_options else None
    return station_options, station_value

get_variable_options(station) ¤

Get valid variable options and default value based on the chosen station.

Parameters:

Name Type Description Default
station str

Code for the chosen station

required

Returns:

Type Description
tuple[list[dict[str, str]], str | None]

tuple[list[dict], str]: Options for the variable dropdown, default value

Source code in measurement\filters.py
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def get_variable_options(station: str) -> tuple[list[dict[str, str]], str | None]:
    """Get valid variable options and default value based on the chosen station.

    Args:
        station (str): Code for the chosen station

    Returns:
        tuple[list[dict], str]: Options for the variable dropdown, default value
    """
    variable_codes = Station.objects.get(station_code=station).variables_list
    variable_dicts = Variable.objects.filter(variable_code__in=variable_codes).values(
        "name", "variable_code"
    )

    variable_options = [
        {
            "label": variable["name"],
            "value": variable["variable_code"],
        }
        for variable in variable_dicts
    ]

    variable_value = variable_options[0]["value"] if variable_options else None
    return variable_options, variable_value

populate_stations_dropdown(station_codes) ¤

Populate the station dropdown based on the list of station codes.

Source code in measurement\dash_apps\daily_validation.py
630
631
632
633
634
635
636
637
638
@app.callback(
    [Output("station_drop", "options"), Output("station_drop", "value")],
    Input("stations_list", "children"),
)
def populate_stations_dropdown(
    station_codes: list[str],
) -> tuple[list[dict[str, str]], str | None]:
    """Populate the station dropdown based on the list of station codes."""
    return get_station_options(station_codes)

populate_variable_dropdown(chosen_station) ¤

Populate the variable dropdown based on the chosen station.

Source code in measurement\dash_apps\daily_validation.py
641
642
643
644
645
646
647
648
649
@app.callback(
    [Output("variable_drop", "options"), Output("variable_drop", "value")],
    Input("station_drop", "value"),
)
def populate_variable_dropdown(
    chosen_station: str,
) -> tuple[list[dict[str, str]], str | None]:
    """Populate the variable dropdown based on the chosen station."""
    return get_variable_options(chosen_station)

reset_validated_days(station, variable, start_date, end_date) ¤

Resets validation and active status for the selected data.

It also deletes the associated report data.

TODO: should this also reset any modified value, minimum or maximum entries?

Parameters:

Name Type Description Default
station str

Station code

required
variable str

Variable code

required
start_date str

Start date

required
end_date str

End date

required
Source code in measurement\validation.py
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
def reset_validated_days(
    station: str, variable: str, start_date: str, end_date: str
) -> None:
    """Resets validation and active status for the selected data.

    It also deletes the associated report data.

    TODO: should this also reset any modified value, minimum or maximum entries?

    Args:
        station (str): Station code
        variable (str): Variable code
        start_date (str): Start date
        end_date (str): End date
    """
    tz = timezone.get_current_timezone()

    # To update we use the exact date range.
    start_date_ = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=tz)
    end_date_ = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=tz)
    Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
        time__date__range=(start_date_.date(), end_date_.date()),
    ).update(is_validated=False, is_active=True)

    # To remove reports we use an extended date range to include the whole month.
    start_date_, end_date_ = reporting.reformat_dates(start_date, end_date)
    reporting.remove_report_data_in_range(station, variable, start_date_, end_date_)

reset_validated_entries(ids) ¤

Resets validation and activation status for the selected data.

TODO: should this also reset any modified value, minimum or maximum entries?

Parameters:

Name Type Description Default
ids list

List of measurement ids to reset.

required
Source code in measurement\validation.py
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
def reset_validated_entries(ids: list) -> None:
    """Resets validation and activation status for the selected data.

    TODO: should this also reset any modified value, minimum or maximum entries?

    Args:
        ids (list): List of measurement ids to reset.
    """
    times: list[datetime] = []
    for _id in ids:
        current = Measurement.objects.get(id=_id)
        current.is_validated = False
        current.is_active = True
        current.save()
        times.append(current.time)

    station = current.station.station_code
    variable = current.variable.variable_code
    start_time, end_time = reporting.reformat_dates(
        to_local_time(min(times)).strftime("%Y-%m-%d"),
        to_local_time(max(times)).strftime("%Y-%m-%d"),
    )

    reporting.remove_report_data_in_range(station, variable, start_time, end_time)

save_validated_days(data) ¤

Saves the validated days to the database and launches the report calculation.

Only the data that is flagged as "validate?" will be saved. The only updated field is is_active. To update the value, maximum or minimum, use save_validated_entries.

Parameters:

Name Type Description Default
data DataFrame

The dataframe with the validated data.

required
Source code in measurement\validation.py
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
def save_validated_days(data: pd.DataFrame) -> None:
    """Saves the validated days to the database and launches the report calculation.

    Only the data that is flagged as "validate?" will be saved. The only updated field
    is is_active. To update the value, maximum or minimum, use save_validated_entries.

    Args:
        data: The dataframe with the validated data.
    """
    tz = timezone.get_current_timezone()
    validate = data[data["validate?"]]
    for _, row in validate.iterrows():
        day = datetime.strptime(row["date"], "%Y-%m-%d").replace(tzinfo=tz)
        Measurement.objects.filter(
            station__station_code=row["station"],
            variable__variable_code=row["variable"],
            time__date=day.date(),
        ).update(is_validated=True, is_active=not row["deactivate?"])

    station = validate["station"].iloc[0]
    variable = validate["variable"].iloc[0]
    start_time = validate["date"].min()
    end_time = validate["date"].max()

    try:
        reporting.launch_reports_calculation(station, variable, start_time, end_time)
    except Exception as e:
        reset_validated_days(station, variable, start_time, end_time)
        raise e

save_validated_entries(data) ¤

Saves the validated data to the database.

Only the data that is flagged as "validate?" will be saved. Possible updated fields are: value, maximum, minimum and is_active.

Parameters:

Name Type Description Default
data DataFrame

The dataframe with the validated data.

required
Source code in measurement\validation.py
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
def save_validated_entries(data: pd.DataFrame) -> None:
    """Saves the validated data to the database.

    Only the data that is flagged as "validate?" will be saved. Possible updated fields
    are: value, maximum, minimum and is_active.

    Args:
        data: The dataframe with the validated data.
    """
    times: list[datetime] = []
    for _, row in data[data["validate?"]].iterrows():
        current = Measurement.objects.get(id=row["id"])
        times.append(current.time)

        update = {"is_validated": True, "is_active": not row["deactivate?"]}
        if current.value != row["value"]:
            update["value"] = row["value"]
        if "maximum" in row and current.maximum != row["maximum"]:
            update["maximum"] = row["maximum"]
        if "minimum" in row and current.minimum != row["minimum"]:
            update["minimum"] = row["minimum"]

        Measurement.objects.filter(id=row["id"]).update(**update)

    tz = timezone.get_current_timezone()
    station = current.station.station_code
    variable = current.variable.variable_code
    start_time = min(times).astimezone(tz).strftime("%Y-%m-%d")
    end_time = max(times).astimezone(tz).strftime("%Y-%m-%d")

    try:
        reporting.launch_reports_calculation(station, variable, start_time, end_time)
    except Exception as e:
        ids = data[data["validate?"]]["id"].tolist()
        reset_validated_entries(ids)
        raise e

set_date_range_min_max(chosen_station, chosen_variable) ¤

Set the default date range and min/max based on the chosen station and variable.

Source code in measurement\dash_apps\daily_validation.py
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
@app.callback(
    [
        Output("date_range_picker", "start_date"),
        Output("date_range_picker", "end_date"),
        Output("minimum_input", "value"),
        Output("maximum_input", "value"),
    ],
    [
        Input("station_drop", "value"),
        Input("variable_drop", "value"),
    ],
)
def set_date_range_min_max(
    chosen_station, chosen_variable
) -> tuple[
    str,
    str,
    Decimal,
    Decimal,
]:
    """Set the default date range and min/max based on the chosen station and
    variable.
    """
    start_date, end_date = get_date_range(chosen_station, chosen_variable)
    min_val, max_val = get_min_max(chosen_station, chosen_variable)
    return start_date, end_date, min_val, max_val

set_detail_date_range(daily_row_data) ¤

Set the min and max date for the detail date picker based on the daily data.

This will run whenever the data is updated.

Parameters:

Name Type Description Default
daily_row_data list[dict]

Data for the daily table

required

Returns:

Type Description
tuple[str, str]

tuple[str, str]: Min date, max date

Source code in measurement\dash_apps\daily_validation.py
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
@app.callback(
    Output("detail-date-picker", "min_date_allowed"),
    Output("detail-date-picker", "max_date_allowed"),
    Input("table_daily", "rowData"),
    prevent_initial_call=True,
)
def set_detail_date_range(daily_row_data) -> tuple[str, str]:
    """Set the min and max date for the detail date picker based on the daily data.

    This will run whenever the data is updated.

    Args:
        daily_row_data (list[dict]): Data for the daily table

    Returns:
        tuple[str, str]: Min date, max date
    """
    if daily_row_data:
        min_date = min(daily_row_data, key=lambda x: x["date"])["date"]
        max_date = max(daily_row_data, key=lambda x: x["date"])["date"]
    else:
        min_date = None
        max_date = None
    return min_date, max_date