daily_validation

measurement.dash_apps.daily_validation

Attributes

DATA_GRANULAR: pd.DataFrame = pd.DataFrame() module-attribute

DATA_SUMMARY: pd.DataFrame = pd.DataFrame() module-attribute

SELECTED_DAY: date | None = None module-attribute

app = DjangoDash('DailyValidation', external_stylesheets=[dbc.themes.BOOTSTRAP, '/static/styles/dashstyle.css']) module-attribute

detail_date_picker = html.Div(children=[html.Div(children=['Open detailed view'], style={'display': 'inline-block', 'padding-right': '5px'}), dcc.DatePickerSingle(id='detail-date-picker', display_format='YYYY-MM-DD', min_date_allowed=None, max_date_allowed=None)], style={'display': 'inline-block', 'width': '50%', 'text-align': 'right'}) module-attribute

filters = html.Div(children=[filters_row1, filters_row2]) module-attribute

filters_row1 = html.Div(children=[html.Div([html.Label('Station:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='station_drop', options=[], value=None)], style={'margin-right': '20px', 'width': '286px', 'display': 'inline-block'}), html.Div([html.Label('Variable:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='variable_drop', options=[], value=None)], style={'margin-right': '20px', 'width': '286px', 'display': 'inline-block'}), html.Div([html.Label('Date Range:', style={'display': 'block', 'font-weight': 'bold'}), dcc.DatePickerRange(id='date_range_picker', display_format='YYYY-MM-DD', start_date=None, end_date=None)], style={'width': '286px', 'display': 'inline-block'})], style={'display': 'flex', 'justify-content': 'flex-start', 'margin-bottom': '10px'}) module-attribute

filters_row2 = html.Div(children=[html.Div([html.Label('Minimum:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Input(id='minimum_input', type='number', value=None)], style={'margin-right': '20px', 'width': '286px'}), html.Div([html.Label('Maximum:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Input(id='maximum_input', type='number', value=None)], style={'margin-right': '20px', 'width': '286px'}), html.Div([html.Label('Validation status:', style={'display': 'block', 'font-weight': 'bold'}), dcc.Dropdown(id='validation_status_drop', options=[{'label': 'Validated', 'value': 'validated'}, {'label': 'Not validated', 'value': 'not_validated'}], value='not_validated')], style={'width': '286px'})], style={'display': 'flex', 'justify-content': 'flex-start'}) module-attribute

menu = html.Div(children=[html.Div(children=[html.Button('Validate', id='save-button')], style={'display': 'inline-block', 'width': '50%'}), detail_date_picker], style={'background-color': '#f0f0f0', 'width': '100%'}) module-attribute

plot = html.Div(children=[dcc.Graph(id='plot', figure=create_empty_plot(), style={'width': '100%'}), dcc.RadioItems(id='plot_radio', options=[{'value': c, 'label': c.capitalize()} for c in ['value', 'maximum', 'minimum']], value='value', style={'width': '100px'}, labelStyle={'display': 'block'})], style={'display': 'flex', 'justify-content': 'space-between', 'height': '400px'}) module-attribute

status_message = html.Div(id='status-message', children=[''], style={'min-height': '20px', 'padding-top': '5px', 'padding-bottom': '10px'}) module-attribute

table_daily = AgGrid(id='table_daily', rowData=[], columnDefs=create_columns_daily(), columnSize='sizeToFit', defaultColDef={'resizable': True, 'sortable': True, 'checkboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelectionFilteredOnly': True}, dashGridOptions={'rowSelection': 'multiple', 'suppressRowClickSelection': True}, selectAll=True, getRowId='params.data.date') module-attribute

table_detail = AgGrid(id='table_detail', rowData=[], columnDefs=create_columns_detail(), columnSize='sizeToFit', defaultColDef={'resizable': True, 'sortable': True, 'checkboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelection': {'function': 'params.column == params.columnApi.getAllDisplayedColumns()[0]'}, 'headerCheckboxSelectionFilteredOnly': True}, dashGridOptions={'rowSelection': 'multiple', 'suppressRowClickSelection': True}, selectAll=True, getRowId='params.data.id') module-attribute

Classes

Variable

Bases: PermissionsBase

A variable with a physical meaning.

Such as precipitation, wind speed, wind direction, soil moisture, including the associated unit. It also includes metadata to help identify what is a reasonable value for the data, to flag outliers and to help with the validation process.

The nature of the variable can be one of the following:

• sum: Cumulative value over a period of time.
• average: Average value over a period of time.
• value: One-off value.

Attributes:

Name	Type	Description
variable_id	AutoField	Primary key.
variable_code	CharField	Code of the variable, eg. airtemperature.
name	CharField	Human-readable name of the variable, eg. Air temperature.
unit	ForeignKey	Unit of the variable.
maximum	DecimalField	Maximum value allowed for the variable.
minimum	DecimalField	Minimum value allowed for the variable.
diff_error	DecimalField	If two sequential values in the time-series data of this variable differ by more than this value, the validation process can mark this with an error flag.
outlier_limit	DecimalField	The statistical deviation for defining outliers, in times the standard deviation (sigma).
null_limit	DecimalField	The max % of null values (missing, caused by e.g. equipment malfunction) allowed for hourly, daily, monthly data. Cumulative values are not deemed trustworthy if the number of missing values in a given period is greater than the null_limit.
nature	CharField	Nature of the variable, eg. if it represents a one-off value, the average over a period of time or the cumulative value over a period

Attributes

is_cumulative: bool property

Return True if the nature of the variable is sum.

Functions

__str__()

Return the string representation of the object.

Source code in variable/models.py

def __str__(self) -> str:
    """Return the string representation of the object."""
    return str(self.name)

clean()

Validate the model fields.

Source code in variable/models.py

def clean(self) -> None:
    """Validate the model fields."""
    if self.maximum < self.minimum:
        raise ValidationError(
            {
                "maximum": "The maximum value must be greater than the minimum "
                "value."
            }
        )
    if not self.variable_code.isidentifier():
        raise ValidationError(
            {
                "variable_code": "The variable code must be a valid Python "
                "identifier. Only letters, numbers and underscores are allowed, and"
                " it cannot start with a number."
            }
        )
    return super().clean()

get_absolute_url()

Get the absolute URL of the object.

Source code in variable/models.py

def get_absolute_url(self) -> str:
    """Get the absolute URL of the object."""
    return reverse("variable:variable_detail", kwargs={"pk": self.pk})

Functions

callbacks(in_submit_clicks, in_save_clicks, in_detail_date, in_plot_radio_value, in_tabs_value, in_station, in_variable, in_start_date, in_end_date, in_minimum, in_maximum, in_daily_selected_rows, in_daily_row_data, in_detail_selected_rows, in_detail_row_data, in_validation_status)

Callbacks for daily validation app

Parameters:

Name	Type	Description	Default
in_submit_clicks	int	Number of times submit-button was clicked	required
in_save_clicks	int	Number of times save-button was clicked	required
in_detail_date	str	Date for detail view	required
in_plot_radio_value	str	Value of plot radio button	required
in_tabs_value	str	Value of tabs	required
in_station	str	Station from filters	required
in_variable	str	Variable from filters	required
in_start_date	str	Start date from filters	required
in_end_date	str	End date from filters	required
in_minimum	float	Minimum from filters	required
in_maximum	float	Maximum from filters	required
in_daily_selected_rows	list[dict]	Selected rows in table_daily	required
in_daily_row_data	list[dict]	Full row data for table_daily	required
in_detail_selected_rows	list[dict]	Selected rows in table_detail	required
in_detail_row_data	list[dict]	Full row data for table_detail	required

Returns:

Name	Type	Description
out_loading_top	no_update	Loading spinner for top
out_loading	no_update	Loading spinner for bottom
out_status	str	Status message
out_plot	Figure	Plot
out_daily_row_data	list[dict]	Data for daily table
out_detail_row_data	list[dict]	Data for detail table
out_daily_selected_rows	list[dict]	Selected rows in daily table
out_detail_selected_rows	list[dict]	Selected rows in detail table
out_tab_detail_disabled	bool	Disabled status for detail tab
out_tab_detail_label	str	Label for detail tab
out_tabs_value	str	Value for tabs
out_save_label	str	Label for save button

Source code in measurement/dash_apps/daily_validation.py

@app.callback(
    [
        Output("loading_top", "children"),
        Output("loading", "children"),
        Output("status-message", "children"),
        Output("plot", "figure"),
        Output("table_daily", "rowData"),
        Output("table_detail", "rowData"),
        Output("table_daily", "selectedRows"),
        Output("table_detail", "selectedRows"),
        Output("tab-detail", "disabled"),
        Output("tab-detail", "label"),
        Output("tabs", "value"),
        Output("save-button", "children"),
    ],
    [
        Input("submit-button", "n_clicks"),
        Input("save-button", "n_clicks"),
        Input("detail-date-picker", "date"),
        Input("plot_radio", "value"),
    ],
    [
        State("tabs", "value"),
        State("station_drop", "value"),
        State("variable_drop", "value"),
        State("date_range_picker", "start_date"),
        State("date_range_picker", "end_date"),
        State("minimum_input", "value"),
        State("maximum_input", "value"),
        State("table_daily", "selectedRows"),
        State("table_daily", "rowData"),
        State("table_detail", "selectedRows"),
        State("table_detail", "rowData"),
        State("validation_status_drop", "value"),
    ],
    prevent_initial_call=True,
)
def callbacks(
    in_submit_clicks: int,
    in_save_clicks: int,
    in_detail_date: str,
    in_plot_radio_value: str,
    in_tabs_value: str,
    in_station: str,
    in_variable: str,
    in_start_date: str,
    in_end_date: str,
    in_minimum: float,
    in_maximum: float,
    in_daily_selected_rows: list[dict],
    in_daily_row_data: list[dict],
    in_detail_selected_rows: list[dict],
    in_detail_row_data: list[dict],
    in_validation_status: str,
) -> tuple[
    dash.no_update,
    dash.no_update,
    str,
    go.Figure,
    list[dict],
    list[dict],
    list[dict],
    list[dict],
    bool,
    str,
    str,
    str,
]:
    """Callbacks for daily validation app

    Args:
        in_submit_clicks (int): Number of times submit-button was clicked
        in_save_clicks (int): Number of times save-button was clicked
        in_detail_date (str): Date for detail view
        in_plot_radio_value (str): Value of plot radio button
        in_tabs_value (str): Value of tabs
        in_station (str): Station from filters
        in_variable (str): Variable from filters
        in_start_date (str): Start date from filters
        in_end_date (str): End date from filters
        in_minimum (float): Minimum from filters
        in_maximum (float): Maximum from filters
        in_daily_selected_rows (list[dict]): Selected rows in table_daily
        in_daily_row_data (list[dict]): Full row data for table_daily
        in_detail_selected_rows (list[dict]): Selected rows in table_detail
        in_detail_row_data (list[dict]): Full row data for table_detail

    Returns:
        out_loading_top (dash.no_update): Loading spinner for top
        out_loading (dash.no_update): Loading spinner for bottom
        out_status (str): Status message
        out_plot (go.Figure): Plot
        out_daily_row_data (list[dict]): Data for daily table
        out_detail_row_data (list[dict]): Data for detail table
        out_daily_selected_rows (list[dict]): Selected rows in daily table
        out_detail_selected_rows (list[dict]): Selected rows in detail table
        out_tab_detail_disabled (bool): Disabled status for detail tab
        out_tab_detail_label (str): Label for detail tab
        out_tabs_value (str): Value for tabs
        out_save_label (str): Label for save button
    """
    global SELECTED_DAY, DATA_SUMMARY, DATA_GRANULAR

    ctx = dash.callback_context
    input_id = ctx.triggered[0]["prop_id"].split(".")[0]

    out_loading_top = dash.no_update
    out_loading = dash.no_update
    out_status = dash.no_update
    out_plot = dash.no_update
    out_daily_row_data = dash.no_update
    out_detail_row_data = dash.no_update
    out_daily_selected_rows = dash.no_update
    out_detail_selected_rows = dash.no_update
    out_tab_detail_disabled = dash.no_update
    out_tab_detail_label = dash.no_update
    out_tabs_value = dash.no_update
    out_save_label = dash.no_update

    data_refresh_required = False
    daily_table_refresh_required = False
    detail_table_refresh_required = False
    daily_table_reset_selection = False
    detail_table_reset_selection = False
    plot_refresh_required = False

    # Button: Submit
    if input_id == "submit-button":
        out_status = ""
        out_save_label = (
            "Validate"
            if in_validation_status == "not_validated"
            else "Reset Validation"
        )
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        daily_table_reset_selection = True
        detail_table_reset_selection = True
        plot_refresh_required = True

    # Button: Save (daily)
    if (
        input_id == "save-button"
        and in_tabs_value == "tab-daily"
        and in_validation_status == "not_validated"
    ):
        selected_dates = {row["date"] for row in in_daily_selected_rows}
        data_to_validate = [
            {
                "date": row["date"].split("T")[0],
                "validate?": True,
                "deactivate?": row["date"] not in selected_dates,
                "station": in_station,
                "variable": in_variable,
            }
            for row in in_daily_row_data
        ]
        save_validated_days(pd.DataFrame.from_records(data_to_validate))
        out_status = "Validation successful"
        data_refresh_required = True
        daily_table_refresh_required = True
        plot_refresh_required = True

    # Button: Save (detail)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-detail"
        and in_validation_status == "not_validated"
    ):
        selected_ids = {row["id"] for row in in_detail_selected_rows}
        data_to_validate = [
            {
                "id": row["id"],
                "validate?": True,
                "deactivate?": row["id"] not in selected_ids,
                "value": row["value"],
                "minimum": row["minimum"],
                "maximum": row["maximum"],
            }
            for row in in_detail_row_data
        ]
        save_validated_entries(pd.DataFrame.from_records(data_to_validate))
        out_status = "Validation successful"
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        plot_refresh_required = True

    # Button: Reset (daily)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-daily"
        and in_validation_status == "validated"
    ):
        reset_validated_days(
            variable=in_variable,
            station=in_station,
            start_date=in_start_date,
            end_date=in_end_date,
        )
        out_status = "Validation reset"
        data_refresh_required = True
        daily_table_refresh_required = True
        daily_table_reset_selection = True
        plot_refresh_required = True

    # Button: Reset (detail)
    elif (
        input_id == "save-button"
        and in_tabs_value == "tab-detail"
        and in_validation_status == "validated"
    ):
        reset_validated_entries(ids=[row["id"] for row in in_detail_row_data])
        out_status = "Validation reset"
        data_refresh_required = True
        daily_table_refresh_required = True
        detail_table_refresh_required = True
        detail_table_reset_selection = True
        plot_refresh_required = True

    # Date picker
    elif input_id == "detail-date-picker":
        new_selected_day = next(
            (
                d.date()
                for d in DATA_SUMMARY["date"]
                if d.strftime("%Y-%m-%d") == in_detail_date
            ),
            None,
        )
        if new_selected_day is not None:
            detail_table_refresh_required = True
            detail_table_reset_selection = True
            out_tab_detail_disabled = False
            out_tab_detail_label = (
                f"Detail of Selected Day ({new_selected_day.strftime('%Y-%m-%d')})"
            )
            out_tabs_value = "tab-detail"
            out_status = ""
            SELECTED_DAY = new_selected_day
        else:
            out_status = "No data for selected day"

    # Plot radio
    elif input_id == "plot_radio":
        plot_refresh_required = True

    # Reload data
    if data_refresh_required:
        DATA_SUMMARY, DATA_GRANULAR = generate_validation_report(
            station=in_station,
            variable=in_variable,
            start_time=in_start_date,
            end_time=in_end_date,
            minimum=Decimal(in_minimum) if in_minimum is not None else None,
            maximum=Decimal(in_maximum) if in_maximum is not None else None,
            is_validated=in_validation_status == "validated",
        )

    # Refresh plot
    if plot_refresh_required:
        if not DATA_GRANULAR.empty:
            out_plot = create_validation_plot(
                data=DATA_GRANULAR,
                variable_name=Variable.objects.get(variable_code=in_variable).name,
                field=in_plot_radio_value,
            )
        else:
            out_plot = create_empty_plot()

    # Refresh daily table
    if daily_table_refresh_required:
        out_daily_row_data = DATA_SUMMARY.to_dict("records")

        # Reset daily table selection
        if daily_table_reset_selection:
            out_daily_selected_rows = out_daily_row_data

    # Refresh detail table
    if detail_table_refresh_required:
        if DATA_GRANULAR.empty:
            out_detail_row_data = []
        else:
            out_detail_row_data = DATA_GRANULAR[
                DATA_GRANULAR.time.dt.date == SELECTED_DAY
            ].to_dict("records")

        # Reset detail table selection
        if detail_table_reset_selection:
            out_detail_selected_rows = [
                row for row in out_detail_row_data if row["is_active"]
            ]

    return (
        out_loading_top,
        out_loading,
        out_status,
        out_plot,
        out_daily_row_data,
        out_detail_row_data,
        out_daily_selected_rows,
        out_detail_selected_rows,
        out_tab_detail_disabled,
        out_tab_detail_label,
        out_tabs_value,
        out_save_label,
    )

create_columns_daily()

Creates columns for Daily Report table

Parameters:

Name	Type	Description	Default
value_columns	list	List of value columns	required

Returns:

Name	Type	Description
list	list	List of columns

Source code in measurement/dash_apps/tables.py

def create_columns_daily() -> list:
    """Creates columns for Daily Report table

    Args:
        value_columns (list): List of value columns

    Returns:
        list: List of columns
    """
    styles = create_style_conditions()

    columns = [
        {
            "valueGetter": {
                "function": "d3.timeParse('%Y-%m-%d')(params.data.date.split('T')[0])"
            },
            "headerName": "Date",
            "filter": "agDateColumnFilter",
            "valueFormatter": {"function": "params.data.date.split('T')[0]"},
            "sort": "asc",
            **styles["date"],
        },
        *[
            {
                "field": c,
                "headerName": c.capitalize(),
                "filter": "agNumberColumnFilter",
                "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
                **styles[c],
            }
            for c in ["value", "minimum", "maximum"]
        ],
        {
            "field": "daily_count_fraction",
            "headerName": "Daily count fraction",
            "filter": "agNumberColumnFilter",
            "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
            **styles["daily_count_fraction"],
        },
        {
            "field": "total_suspicious_entries",
            "headerName": "Suspicious entries",
            "filter": "agNumberColumnFilter",
            **styles["total_suspicious_entries"],
        },
    ]
    return columns

create_columns_detail()

Creates columns for Detail table

Parameters:

Name	Type	Description	Default
value_columns	list	List of value columns	required

Returns:

Name	Type	Description
list	list	List of columns

Source code in measurement/dash_apps/tables.py

def create_columns_detail() -> list:
    """Creates columns for Detail table

    Args:
        value_columns (list): List of value columns

    Returns:
        list: List of columns
    """
    styles = create_style_conditions()

    columns = [
        {
            "field": "id",
            "headerName": "Measurement ID",
            "filter": "agNumberColumnFilter",
        },
        {
            "field": "time",
            "valueFormatter": {"function": "params.value.split('T')[1].split('+')[0]"},
            "headerName": "Time",
            "editable": True,
            "sort": "asc",
            **styles["time"],
        },
        *[
            {
                "field": c,
                "headerName": c.capitalize(),
                "filter": "agNumberColumnFilter",
                "editable": True,
                "valueFormatter": {"function": "d3.format(',.2f')(params.value)"},
                **styles[c],
            }
            for c in ["value", "minimum", "maximum"]
        ],
    ]
    return columns

create_empty_plot()

Creates empty plot

Returns:

Type	Description
scatter	px.Scatter: Plot

Source code in measurement/dash_apps/plots.py

def create_empty_plot() -> px.scatter:
    """Creates empty plot

    Returns:
        px.Scatter: Plot
    """
    fig = px.scatter(title="No data to plot")
    fig.update_layout(
        autosize=True,
        margin=dict(
            l=50,
            r=20,
            b=0,
            t=50,
        ),
        title_font=dict(
            size=14,
        ),
    )
    return fig

create_validation_plot(data, variable_name, field)

Creates plot for Validation app

Parameters:

Name	Type	Description	Default
data	DataFrame	Data	required
variable_name	str	Variable name	required
field	str	'value', 'minimum' or 'maximum'	required

Returns:

Type	Description
scatter	px.Scatter: Plot

Source code in measurement/dash_apps/plots.py

def create_validation_plot(
    data: pd.DataFrame, variable_name: str, field: str
) -> px.scatter:
    """Creates plot for Validation app

    Args:
        data (pd.DataFrame): Data
        variable_name (str): Variable name
        field (str): 'value', 'minimum' or 'maximum'

    Returns:
        px.Scatter: Plot
    """

    def status(row):
        if not row["is_validated"]:
            return "Not validated"
        if row["is_active"]:
            return "Active"
        return "Inactive"

    color_map = {
        "Active": "#00CC96",
        "Inactive": "#636EFA",
        "Not validated": "black",
    }

    fig = px.scatter(
        data,
        x="time",
        y=field,
        color=data.apply(status, axis=1),
        color_discrete_map=color_map,
        labels={"time": "Date", field: f"{variable_name} ({field.capitalize()})"},
    )

    fig.update_traces(marker=dict(size=3))
    fig.update_layout(
        legend=dict(
            title=dict(text="Status", font=dict(size=12)),
            x=1,
            y=1,
            xanchor="auto",
            yanchor="auto",
        ),
        autosize=True,
        margin=dict(
            l=50,
            r=20,
            b=0,
            t=50,
        ),
    )

    return fig

generate_validation_report(station, variable, start_time, end_time, maximum, minimum, is_validated=False)

Generates a report of the data.

Parameters:

Name	Type	Description	Default
station	str	Station of interest.	required
variable	str	Variable of interest.	required
start_time	str	Start time.	required
end_time	str	End time.	required
maximum	Decimal	The maximum allowed value.	required
minimum	Decimal	The minimum allowed value.	required
is_validated	bool	Whether to retrieve validated or non-validated data.	False

Returns:

Type	Description
tuple[DataFrame, DataFrame]	A tuple with the summary report and the granular report.

Source code in measurement/validation.py

def generate_validation_report(
    station: str,
    variable: str,
    start_time: str,
    end_time: str,
    maximum: Decimal,
    minimum: Decimal,
    is_validated: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame]:
    """Generates a report of the data.

    Args:
        station: Station of interest.
        variable: Variable of interest.
        start_time: Start time.
        end_time: End time.
        maximum: The maximum allowed value.
        minimum: The minimum allowed value.
        is_validated: Whether to retrieve validated or non-validated data.

    Returns:
        A tuple with the summary report and the granular report.
    """
    period = Station.objects.get(station_code=station).delta_t.delta_t
    var = Variable.objects.get(variable_code=variable)

    data = get_data_to_validate(station, variable, start_time, end_time, is_validated)
    if data.empty:
        return pd.DataFrame(), pd.DataFrame()

    suspicious = flag_suspicious_data(data, maximum, minimum, period, var.diff_error)
    summary = generate_daily_summary(
        data, suspicious, period, var.null_limit, var.is_cumulative
    )
    granular = pd.concat([data, suspicious], axis=1)
    return summary, granular

get_date_range(station, variable)

Get the date range covered by a chosen station and variable.

Parameters:

Name	Type	Description	Default
station	str	Code for the chosen station	required
variable	str	Code for the chosen variable	required

Returns:

Type	Description
tuple[str, str]	tuple[str, str]: Start date, end date

Source code in measurement/filters.py

def get_date_range(station: str, variable: str) -> tuple[str, str]:
    """Get the date range covered by a chosen station and variable.

    Args:
        station (str): Code for the chosen station
        variable (str): Code for the chosen variable

    Returns:
        tuple[str, str]: Start date, end date
    """
    filter_vals = Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
    ).aggregate(
        first_date=Min("time"),
        last_date=Max("time"),
    )

    first_date = (
        filter_vals["first_date"].strftime("%Y-%m-%d")
        if filter_vals["first_date"]
        else None
    )
    last_date = (
        filter_vals["last_date"].strftime("%Y-%m-%d")
        if filter_vals["last_date"]
        else None
    )

    return first_date, last_date

get_min_max(station, variable)

Get the min and max of the data for a chosen station and variable.

Parameters:

Name	Type	Description	Default
station	str	Code for the chosen station	required
variable	str	Code for the chosen variable	required

Returns:

Type	Description
tuple[Decimal, Decimal]	tuple[Decimal, Decimal]: Min value, max value

Source code in measurement/filters.py

def get_min_max(
    station, variable
) -> tuple[
    Decimal,
    Decimal,
]:
    """Get the min and max of the data for a chosen station and variable.

    Args:
        station (str): Code for the chosen station
        variable (str): Code for the chosen variable

    Returns:
        tuple[Decimal, Decimal]: Min value, max value
    """
    filter_vals = Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
    ).aggregate(
        min_value=Min("minimum"),
        max_value=Max("maximum"),
    )

    min_value = filter_vals["min_value"] if filter_vals["min_value"] else None
    max_value = filter_vals["max_value"] if filter_vals["max_value"] else None

    return min_value, max_value

get_station_options(station_codes)

Get valid station options and default value based on permissions and data availability.

Parameters:

Name	Type	Description	Default
station_codes	list[str]	List of station codes based on permissions	required

Returns:

Type	Description
tuple[list[dict[str, str]], str | None]	tuple[list[dict], str]: Options for the station dropdown, default value

Source code in measurement/filters.py

def get_station_options(
    station_codes: list[str],
) -> tuple[list[dict[str, str]], str | None]:
    """Get valid station options and default value based on permissions and data
    availability.

    Args:
        station_codes (list[str]): List of station codes based on permissions

    Returns:
        tuple[list[dict], str]: Options for the station dropdown, default value
    """
    stations_with_measurements = Measurement.objects.values_list(
        "station__station_code", flat=True
    ).distinct()

    station_options = [
        {"label": station_code, "value": station_code}
        for station_code in station_codes
        if station_code in stations_with_measurements
    ]
    station_value = station_options[0]["value"] if station_options else None
    return station_options, station_value

get_variable_options(station)

Get valid variable options and default value based on the chosen station.

Parameters:

Name	Type	Description	Default
station	str	Code for the chosen station	required

Returns:

Type	Description
tuple[list[dict], str]	tuple[list[dict], str]: Options for the variable dropdown, default value

Source code in measurement/filters.py

def get_variable_options(station: str) -> tuple[list[dict], str]:
    """Get valid variable options and default value based on the chosen station.

    Args:
        station (str): Code for the chosen station

    Returns:
        tuple[list[dict], str]: Options for the variable dropdown, default value
    """
    variable_dicts = (
        Measurement.objects.filter(station__station_code=station)
        .values("variable__name", "variable__variable_code")
        .distinct()
    )

    variable_options = [
        {
            "label": variable["variable__name"],
            "value": variable["variable__variable_code"],
        }
        for variable in variable_dicts
    ]
    variable_value = variable_options[0]["value"] if variable_options else None
    return variable_options, variable_value

populate_stations_dropdown(station_codes)

Populate the station dropdown based on the list of station codes.

Source code in measurement/dash_apps/daily_validation.py

@app.callback(
    [Output("station_drop", "options"), Output("station_drop", "value")],
    Input("stations_list", "children"),
)
def populate_stations_dropdown(
    station_codes: list[str],
) -> tuple[list[dict[str, str]], str | None]:
    """Populate the station dropdown based on the list of station codes."""
    return get_station_options(station_codes)

populate_variable_dropdown(chosen_station)

Populate the variable dropdown based on the chosen station.

Source code in measurement/dash_apps/daily_validation.py

@app.callback(
    [Output("variable_drop", "options"), Output("variable_drop", "value")],
    Input("station_drop", "value"),
)
def populate_variable_dropdown(
    chosen_station: str,
) -> tuple[list[dict[str, str]], str | None]:
    """Populate the variable dropdown based on the chosen station."""
    return get_variable_options(chosen_station)

reset_validated_days(station, variable, start_date, end_date)

Resets validation and active status for the selected data.

It also deletes the associated report data.

TODO: should this also reset any modified value, minimum or maximum entries?

Parameters:

Name	Type	Description	Default
station	str	Station code	required
variable	str	Variable code	required
start_date	str	Start date	required
end_date	str	End date	required

Source code in measurement/validation.py

def reset_validated_days(
    station: str, variable: str, start_date: str, end_date: str
) -> None:
    """Resets validation and active status for the selected data.

    It also deletes the associated report data.

    TODO: should this also reset any modified value, minimum or maximum entries?

    Args:
        station (str): Station code
        variable (str): Variable code
        start_date (str): Start date
        end_date (str): End date
    """
    tz = zoneinfo.ZoneInfo(Station.objects.get(station_code=station).timezone)

    # To update we use the exact date range.
    start_date_ = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=tz)
    end_date_ = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=tz)
    Measurement.objects.filter(
        station__station_code=station,
        variable__variable_code=variable,
        time__date__range=(start_date_.date(), end_date_.date()),
    ).update(is_validated=False, is_active=True)

    # To remove reports we use an extendaed date range to include the whole month.
    start_date_, end_date_ = reporting.reformat_dates(station, start_date, end_date)
    reporting.remove_report_data_in_range(station, variable, start_date_, end_date_)

reset_validated_entries(ids)

Resets validation and activation status for the selected data.

TODO: should this also reset any modified value, minimum or maximum entries?

Parameters:

Name	Type	Description	Default
ids	list	List of measurement ids to reset.	required

Source code in measurement/validation.py

def reset_validated_entries(ids: list) -> None:
    """Resets validation and activation status for the selected data.

    TODO: should this also reset any modified value, minimum or maximum entries?

    Args:
        ids (list): List of measurement ids to reset.
    """
    times = []
    for _id in ids:
        current = Measurement.objects.get(id=_id)
        current.is_validated = False
        current.is_active = True
        current.save()
        times.append(current.time)

    station = current.station.station_code
    variable = current.variable.variable_code
    start_time, end_time = reporting.reformat_dates(
        station, min(times).strftime("%Y-%m-%d"), max(times).strftime("%Y-%m-%d")
    )

    reporting.remove_report_data_in_range(station, variable, start_time, end_time)

save_validated_days(data)

Saves the validated days to the database and launches the report calculation.

Only the data that is flagged as "validate?" will be saved. The only updated field is is_active. To update the value, maximum or minimum, use save_validated_entries.

Parameters:

Name	Type	Description	Default
data	DataFrame	The dataframe with the validated data.	required

Source code in measurement/validation.py

def save_validated_days(data: pd.DataFrame) -> None:
    """Saves the validated days to the database and launches the report calculation.

    Only the data that is flagged as "validate?" will be saved. The only updated field
    is is_active. To update the value, maximum or minimum, use save_validated_entries.

    Args:
        data: The dataframe with the validated data.
    """
    validate = data[data["validate?"]]
    for _, row in validate.iterrows():
        Measurement.objects.filter(
            station__station_code=row["station"],
            variable__variable_code=row["variable"],
            time__date=row["date"],
        ).update(is_validated=True, is_active=not row["deactivate?"])

    station = validate["station"].iloc[0]
    variable = validate["variable"].iloc[0]
    start_time = validate["date"].min()
    end_time = validate["date"].max()

    reporting.launch_reports_calculation(station, variable, start_time, end_time)

save_validated_entries(data)

Saves the validated data to the database.

Only the data that is flagged as "validate?" will be saved. Possible updated fields are: value, maximum, minimum and is_active.

Parameters:

Name	Type	Description	Default
data	DataFrame	The dataframe with the validated data.	required

Source code in measurement/validation.py

def save_validated_entries(data: pd.DataFrame) -> None:
    """Saves the validated data to the database.

    Only the data that is flagged as "validate?" will be saved. Possible updated fields
    are: value, maximum, minimum and is_active.

    Args:
        data: The dataframe with the validated data.
    """
    times: list[datetime] = []
    for _, row in data[data["validate?"]].iterrows():
        current = Measurement.objects.get(id=row["id"])
        times.append(current.time)

        update = {"is_validated": True, "is_active": not row["deactivate?"]}
        if current.value != row["value"]:
            update["value"] = row["value"]
        if "maximum" in row and current.maximum != row["maximum"]:
            update["maximum"] = row["maximum"]
        if "minimum" in row and current.minimum != row["minimum"]:
            update["minimum"] = row["minimum"]

        Measurement.objects.filter(id=row["id"]).update(**update)

    tz = zoneinfo.ZoneInfo(current.station.timezone)
    station = current.station.station_code
    variable = current.variable.variable_code
    start_time = min(times).astimezone(tz).strftime("%Y-%m-%d")
    end_time = max(times).astimezone(tz).strftime("%Y-%m-%d")

    reporting.launch_reports_calculation(station, variable, start_time, end_time)

set_date_range_min_max(chosen_station, chosen_variable)

Set the default date range and min/max based on the chosen station and variable.

Source code in measurement/dash_apps/daily_validation.py

@app.callback(
    [
        Output("date_range_picker", "start_date"),
        Output("date_range_picker", "end_date"),
        Output("minimum_input", "value"),
        Output("maximum_input", "value"),
    ],
    [
        Input("station_drop", "value"),
        Input("variable_drop", "value"),
    ],
)
def set_date_range_min_max(
    chosen_station, chosen_variable
) -> tuple[
    str,
    str,
    Decimal,
    Decimal,
]:
    """Set the default date range and min/max based on the chosen station and
    variable.
    """
    start_date, end_date = get_date_range(chosen_station, chosen_variable)
    min_val, max_val = get_min_max(chosen_station, chosen_variable)
    return start_date, end_date, min_val, max_val

set_detail_date_range(daily_row_data)

Set the min and max date for the detail date picker based on the daily data.

This will run whenever the data is updated.

Parameters:

Name	Type	Description	Default
daily_row_data	list[dict]	Data for the daily table	required

Returns:

Type	Description
tuple[str, str]	tuple[str, str]: Min date, max date

Source code in measurement/dash_apps/daily_validation.py

@app.callback(
    Output("detail-date-picker", "min_date_allowed"),
    Output("detail-date-picker", "max_date_allowed"),
    Input("table_daily", "rowData"),
    prevent_initial_call=True,
)
def set_detail_date_range(daily_row_data) -> tuple[str, str]:
    """Set the min and max date for the detail date picker based on the daily data.

    This will run whenever the data is updated.

    Args:
        daily_row_data (list[dict]): Data for the daily table

    Returns:
        tuple[str, str]: Min date, max date
    """
    if daily_row_data:
        min_date = min(daily_row_data, key=lambda x: x["date"])["date"]
        max_date = max(daily_row_data, key=lambda x: x["date"])["date"]
    else:
        min_date = None
        max_date = None
    return min_date, max_date