views

importing.views

Attributes

logger = logging.getLogger(__name__) module-attribute

Classes

DataImport

Bases: PermissionsBase

Model to store the data imports.

This model stores the data imports, which are, often, files with data that are uploaded to the system. The data is then processed asynchronously and stored in the database.

Attributes:

Name	Type	Description
station	ForeignKey	Station to which the data belongs.
format	ForeignKey	Format of the data.
rawfile	FileField	File with the data to be imported.
date	DateTimeField	Date of submission of the data.
start_date	DateTimeField	Start date of the data.
end_date	DateTimeField	End date of the data.
records	IntegerField	Number of records in the data.
observations	TextField	Notes or observations about the data.
status	TextField	Status of the import.
log	TextField	Log of the data ingestion, indicating any errors.

Functions

clean()

Validate information and uploads the measurement data.

Source code in importing/models.py

def clean(self) -> None:
    """Validate information and uploads the measurement data."""
    tz = self.station.timezone
    if not tz:
        raise ValidationError("Station must have a timezone set.")

    if self.origin.origin == "Thingsboard" and not self.format.thingsboard:
        raise ValidationError("Ensure a Thingsboard-specific format is specified.")

DataImportCreateView

Bases: CustomCreateView

View to create a data import.

DataImportDeleteView

Bases: CustomDeleteView

View to delete a data import.

DataImportDetailSerializer

Bases: ModelSerializer

Detailed serializer used when the requesting user is the owner.

Includes the ingestion log so clients can debug failures.

DataImportDetailView

Bases: CustomDetailView

View to view a data import.

DataImportEditView

Bases: CustomEditView

View to edit a data import.

Functions

form_valid(form)

Reprocess if a new file is uploaded or the reprocess button is selected.

Source code in importing/views.py

def form_valid(self, form):
    """Reprocess if a new file is uploaded or the reprocess button is selected."""
    reprocess = self.request.POST.get("action") == "reprocess"
    new_file = self.object.pk and "rawfile" in form.changed_data
    if reprocess or new_file:
        self.object.status = "N"
        self.object.save()

    return super().form_valid(form)

DataImportFilter

Bases: FilterSet

DataImportListView

Bases: CustomTableView

View to list all data imports.

DataImportTable

Bases: Table

DataImportUploadAPIView

Bases: APIView

API endpoint for uploading data files for import.

Access is controlled via object-level permissions using django-guardian. Users can only upload data for Station objects for which they have the change_station permission assigned.

Functions

get_permitted_stations(user)

Get stations the user has permission to import data for.

Source code in importing/views.py

def get_permitted_stations(self, user):
    """Get stations the user has permission to import data for."""
    return get_objects_for_user(user, "change_station", klass=Station)

post(request)

Upload a data file and create a data import.

Source code in importing/views.py

@extend_schema(
    summary="Upload data file for import",
    description="""
    Upload a data file to create a new data import.

    **Permissions**: Users can only upload data for stations they have
    `change_station` permission for.

    **File Upload**: Send as multipart/form-data with all parameters in the body.
    """,
    request={
        "multipart/form-data": {
            "type": "object",
            "properties": {
                "station": {
                    "type": "string",
                    "description": "Station code",
                },
                "format": {
                    "type": "integer",
                    "description": "Format ID",
                },
                "rawfile": {
                    "type": "string",
                    "format": "binary",
                    "description": "Data file to upload",
                },
                "visibility": {
                    "type": "string",
                    "enum": ["public", "private"],
                    "default": "private",
                    "description": "Visibility level",
                },
                "observations": {
                    "type": "string",
                    "description": "Additional observations",
                },
            },
            "required": ["station", "format", "rawfile"],
        }
    },
    responses={
        201: OpenApiResponse(
            response=DataImportUploadResponseSerializer,
            description="Data import created successfully",
            examples=[
                OpenApiExample(
                    "Successful upload",
                    value={
                        "data_import_id": 123,
                        "station": "CAR_02_HC_01",
                        "format": 47,
                        "rawfile": "/media/imports/data_2024.csv",
                        "visibility": "private",
                        "date": "2024-12-09T10:30:00Z",
                        "status": "N",
                        "status_display": "Not queued",
                    },
                )
            ],
        ),
        400: OpenApiResponse(description="Invalid request parameters or file"),
        403: OpenApiResponse(
            description="User does not have permission to upload data."
        ),
        404: OpenApiResponse(description="Station or format not found"),
    },
    tags=["importing"],
)
def post(self, request):
    """Upload a data file and create a data import."""
    # Get the uploaded file
    rawfile = request.FILES.get("rawfile")
    if not rawfile:
        return Response(
            {"rawfile": ["No file was submitted."]},
            status=status.HTTP_400_BAD_REQUEST,
        )

    # Build data dict for serializer
    data = {
        "station": request.data.get("station"),
        "format": request.data.get("format"),
        "visibility": request.data.get("visibility", "private"),
        "observations": request.data.get("observations", ""),
        "rawfile": rawfile,
    }

    # Validate request parameters
    serializer = DataImportUploadRequestSerializer(data=data)
    if not serializer.is_valid():
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    validated_data = serializer.validated_data
    station = validated_data["station"]

    # Check user permissions for this station
    permitted_stations = self.get_permitted_stations(request.user)
    if station not in permitted_stations:
        return Response(
            {
                "detail": "You do not have permission to upload data "
                f"for station '{station.station_code}'."
            },
            status=status.HTTP_403_FORBIDDEN,
        )

    # Create the data import
    try:
        data_import = DataImport.objects.create(
            station=station,
            format=validated_data["format"],
            origin=ImportOrigin.objects.get_or_create(origin="api")[0],
            visibility=validated_data["visibility"],
            observations=validated_data.get("observations", ""),
            rawfile=validated_data["rawfile"],
            owner=request.user,
        )
    except Exception as e:
        logging.exception(f"Error creating data import: {e}")
        return Response(
            {"detail": "An internal error occurred while creating data import."},
            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
        )

    # Return the created import
    response_serializer = DataImportUploadResponseSerializer(
        data_import, context={"request": request}
    )
    return Response(response_serializer.data, status=status.HTTP_201_CREATED)

DataImportUploadRequestSerializer

Bases: ModelSerializer

Serializer for data import upload request.

DataImportUploadResponseSerializer

Bases: ModelSerializer

Serializer for data import upload response.

DataIngestionQueryView

Bases: APIView

API endpoint for querying data ingestion status/list.

• No data_import_id query param: return list of imports the user may view.
• With data_import_id: only the owner may request detailed info (including log). Non-owners receive 403 for direct PK detail requests; they should use the list endpoint.

Functions

get(request)

Return list or owner-only detail for data import ingestion status.

Source code in importing/views.py

@extend_schema(
    summary="Query data ingestion status",
    description="""
    - If `data_import_id` query parameter is provided, the endpoint returns the
      details for that DataImport **only if the requesting user is the owner**.
      Owners receive the full detail including the ingestion `log`.
    - If no `data_import_id` is provided, the endpoint returns a list of
      DataImport objects the user can view, ordered by submission date.
    """,
    parameters=[
        OpenApiParameter(
            name="data_import_id",
            type=OpenApiTypes.INT,
            location=OpenApiParameter.QUERY,
            required=False,
            description="Primary key of a DataImport to return detailed info.",
        ),
    ],
    responses={
        200: OpenApiResponse(
            description="Data ingestion status retrieved successfully"
        ),
        400: OpenApiResponse(description="Invalid request parameters"),
        403: OpenApiResponse(
            description="Only the owner may request a specific data import by PK"
        ),
        404: OpenApiResponse(description="Data import not found"),
    },
    tags=["importing"],
)
def get(self, request):
    """Return list or owner-only detail for data import ingestion status."""
    data_import_id = request.query_params.get("data_import_id")

    if not data_import_id:
        # Return list of DataImport objects the user can view
        permitted_qs = get_objects_for_user(
            request.user, "view_dataimport", klass=DataImport
        ).order_by("-date")
        serializer = DataImportUploadResponseSerializer(
            permitted_qs, many=True, context={"request": request}
        )
        return Response(serializer.data, status=status.HTTP_200_OK)

    # If a PK is provided, fetch the object and enforce owner-only access.
    try:
        data_import = DataImport.objects.get(pk=data_import_id)
    except DataImport.DoesNotExist:
        return Response(
            {"detail": "Data import not found."}, status=status.HTTP_404_NOT_FOUND
        )

    # Only the owner may request detailed info by PK
    if data_import.owner != request.user:
        return Response(
            {
                "detail": "Only the owner may request this data import by primary key."  # noqa: E501
            },
            status=status.HTTP_403_FORBIDDEN,
        )

    # Owner may view full details (including log)
    serializer = DataImportDetailSerializer(
        data_import, context={"request": request}
    )
    return Response(serializer.data, status=status.HTTP_200_OK)

ImportOrigin

Bases: Model

Class that contains the origin of the data import, eg. file, API, etc.

Functions

get_default() classmethod

Get default import origin, 'file'.

It should exist, as it is created in a data migration, but just in case it is not, we use get_or_create.

Source code in importing/models.py

@classmethod
def get_default(cls) -> ImportOrigin:
    """Get default import origin, 'file'.

    It should exist, as it is created in a data migration, but just in case it
    is not, we use get_or_create.
    """
    obj, _ = cls.objects.get_or_create(origin="file")
    return obj.pk

MapLayerCreateView

Bases: CustomCreateView

View to create a map layer import.

MapLayerDeleteView

Bases: CustomDeleteView

View to delete a map layer import.

MapLayerDetailView

Bases: CustomDetailView

View to view a data import.

MapLayerEditView

Bases: CustomEditView

View to edit a map layer imports.

MapLayerImport

Bases: PermissionsBase

Functions

clean()

Validate the uploaded GeoTIFF and its transformed lon/lat bounds.

Source code in importing/models.py

def clean(self) -> None:
    """Validate the uploaded GeoTIFF and its transformed lon/lat bounds."""

    if not self.file:
        return

    import rasterio
    from rasterio.warp import transform_bounds

    file_obj = self.file.file
    try:
        with rasterio.MemoryFile(file_obj.read()) as memfile:
            with memfile.open() as dataset:
                if dataset.count == 0:
                    raise ValidationError(
                        {"file": "File contains no raster bands."}
                    )
                if dataset.crs is None:
                    raise ValidationError(
                        {"file": "File has no coordinate reference system (CRS)."}
                    )

                left, bottom, right, top = transform_bounds(
                    dataset.crs,
                    "EPSG:4326",
                    dataset.bounds.left,
                    dataset.bounds.bottom,
                    dataset.bounds.right,
                    dataset.bounds.top,
                )

                if not (-180 <= left <= 180 and -180 <= right <= 180):
                    raise ValidationError(
                        {
                            "file": (
                                "GeoTIFF longitude values are out of range "
                                "[-180, 180]."
                            )
                        }
                    )
                if not (-90 <= bottom <= 90 and -90 <= top <= 90):
                    raise ValidationError(
                        {
                            "file": (
                                "GeoTIFF latitude values are out of range "
                                "[-90, 90]."
                            )
                        }
                    )
    except ValidationError:
        raise
    except Exception as e:
        raise ValidationError(
            {"file": f"Could not open as a valid GeoTIFF or read coordinates: {e}"}
        )
    finally:
        file_obj.seek(0)

MapLayerImportFilter

Bases: FilterSet

MapLayerImportTable

Bases: Table

MapLayerListView

Bases: CustomTableView

View to list all map layer imports.

Station

Bases: PermissionsBase

Main representation of a station, including several metadata.

Attributes:

Name	Type	Description
visibility	str	Visibility level of the object, including an "internal" option.
station_id	int	Primary key.
station_code	str	Unique code for the station.
station_name	str	Brief description of the station.
station_type	StationType	Type of the station.
country	Country	Country where the station is located.
region	Region	Region within the Country where the station is located.
ecosystem	Ecosystem	Ecosystem associated with the station.
institution	Institution	Institutional partner responsible for the station.
place_basin	PlaceBasin	Place-Basin association.
station_state	bool	Is the station operational?
timezone	str	Timezone of the station.
station_latitude	Decimal	Latitude of the station, in degrees [-90 to 90].
station_longitude	Decimal	Longitude of the station, in degrees [-180 to 180].
station_altitude	int	Altitude of the station.
influence_km	Decimal	Area of influence in km2.
station_file	ImageField	Photography of the station.
station_external	bool	Is the station external?
variables	str	Comma-separated list of variables measured by the station.

Attributes

variables_list property

Return the list of variables measured by the station.

Only variables with data in the database are returned.

Returns:

Type	Description
list[str]	list[str]: List of variables measured by the station.

Functions

__str__()

Return the station code.

Source code in station/models.py

def __str__(self) -> str:
    """Return the station code."""
    return str(self.station_code)

get_absolute_url()

Return the absolute url of the station.

Source code in station/models.py

def get_absolute_url(self) -> str:
    """Return the absolute url of the station."""
    return reverse("station:station_detail", kwargs={"pk": self.pk})

set_object_permissions()

Set object-level permissions.

This method is called by the save method of the model to set the object-level permissions based on the visibility level of the object. In addition to the standard permissions for the station, the view_measurements permission is set which controls who can view the measurements associated to the station.

Source code in station/models.py

def set_object_permissions(self) -> None:
    """Set object-level permissions.

    This method is called by the save method of the model to set the object-level
    permissions based on the visibility level of the object. In addition to the
    standard permissions for the station, the view_measurements permission is set
    which controls who can view the measurements associated to the station.
    """
    super().set_object_permissions()

    standard_group = Group.objects.get(name="Standard")
    anonymous_user = get_anonymous_user()

    # Assign view_measurements permission based on permissions level
    if self.visibility == "public":
        assign_perm("view_measurements", standard_group, self)
        assign_perm("view_measurements", anonymous_user, self)
        if self.owner:
            remove_perm("view_measurements", self.owner, self)
    elif self.visibility == "internal":
        assign_perm("view_measurements", standard_group, self)
        remove_perm("view_measurements", anonymous_user, self)
        if self.owner:
            remove_perm("view_measurements", self.owner, self)
    elif self.visibility == "private":
        remove_perm("view_measurements", standard_group, self)
        remove_perm("view_measurements", anonymous_user, self)
        if self.owner:
            assign_perm("view_measurements", self.owner, self)

ThingsBoardImportMapListView

Bases: CustomTableView

ThingsboardDataRetrievalForm(*args, user=None, **kwargs)

Bases: Form

Form to select ThingsboardImportMap and date range for data retrieval.

Source code in importing/forms.py

def __init__(self, *args, user=None, **kwargs):
    super().__init__(*args, **kwargs)
    if user is not None:
        self.fields["thingsboard_map"].queryset = get_objects_for_user(
            user,
            "importing.view_thingsboardimportmap",
            klass=ThingsboardImportMap,
        )
        self.fields["format"].queryset = get_objects_for_user(
            user,
            "formatting.view_format",
            klass=Format,
        ).order_by("name")

ThingsboardDataRetrievalView

Bases: LoginRequiredMixin, FormView

ThingsboardImportMap

Bases: PermissionsBase

Model to store Thingsboard device mappings to station variables.

This model maps Thingsboard devices to specific variables at stations, allowing data from IoT devices to be imported and associated with the correct station and variable combinations.

Attributes:

Name	Type	Description
tb_variable	CharField	Name of the variable in Thingsboard.
variable	ForeignKey	The existing variable in Paricia associated with this mapping.
tb_device_name	CharField	The name of the device in Thingsboard.
station	ForeignKey	The name of the corresponding station in Paricia.

Functions

clean()

Validate that the variable is valid for the station.

Source code in importing/models.py

def clean(self) -> None:
    """Validate that the variable is valid for the station."""
    try:
        station = self.station
    except ThingsboardImportMap.station.RelatedObjectDoesNotExist:
        raise ValidationError({"station": "Station is required."})

    try:
        variable = self.variable
    except ThingsboardImportMap.variable.RelatedObjectDoesNotExist:
        raise ValidationError({"variable": "Variable is required."})

    # Check if the variable is valid for the station through SensorInstallation
    if not SensorInstallation.objects.filter(
        variable=variable, station=station
    ).exists():
        raise ValidationError(
            {
                "variable": f"Variable '{variable}' is not configured for"
                f" station '{station}' via a sensor installation."
            }
        )

ThingsboardImportMapCreateView

Bases: CustomCreateView

ThingsboardImportMapDeleteView

Bases: CustomDeleteView

ThingsboardImportMapDetailView

Bases: CustomDetailView

ThingsboardImportMapEditView

Bases: CustomEditView

ThingsboardImportMapFilter

Bases: FilterSet

ThingsboardImportMapTable

Bases: Table

Functions

retrieve_thingsboard_data(token, customer_id, tb_device_name, variable, start_ts, end_ts)

Retrieves data from ThingsBoard for a given device and variable.

Saves the response to a JSON file in the media directory.

Source code in importing/utils.py

def retrieve_thingsboard_data(
    token: str,
    customer_id: str,
    tb_device_name: str,
    variable: str,
    start_ts: int,
    end_ts: int,
) -> dict:
    """Retrieves data from ThingsBoard for a given device and variable.

    Saves the response to a JSON file in the media directory.
    """

    tb_device_id = retrieve_thingsboard_device_id(token, customer_id, tb_device_name)
    logger.debug(
        f"Retrieving ThingsBoard data for device {tb_device_id}, variable {variable}"
    )
    assert settings.TB_TIMESERIES_URL is not None
    url = settings.TB_TIMESERIES_URL.format(tb_device_id=tb_device_id)
    headers = {"X-Authorization": f"Bearer {token}"}
    params: dict[str, str | int] = {
        "limit": 10000,
        "agg": "NONE",
        "keys": variable,
        "startTs": start_ts,
        "endTs": end_ts,
    }

    response = requests.get(url, headers=headers, params=params)

    if response.status_code == 200:
        return response.json()
    else:
        raise Exception(
            f"Failed to retrieve data: {response.status_code} - {response.text}"
        )