From 69cbfa2d3f0a8e2eff99304b6e456245817c03e7 Mon Sep 17 00:00:00 2001 From: Ian Roddis <31021769+iroddis@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:15:35 -0300 Subject: [PATCH] Adding scope import --- .../d1e2f3a4b5c6_add_scope_infos_table.py | 38 ++ app/api/router.py | 4 + app/api/scope_import.py | 53 +++ app/crud/augmented_part.py | 7 + app/crud/scope_info.py | 56 +++ app/crud/scope_model.py | 12 + app/models/__init__.py | 2 + app/models/scope_info.py | 34 ++ app/schemas/scope_import.py | 17 + app/schemas/scope_info.py | 15 + app/services/scope_import.py | 427 ++++++++++++++++++ app/templates/_nav.html | 8 + app/templates/admin/_scope_import_result.html | 41 ++ app/templates/admin/scope_import.html | 37 ++ app/web/admin.py | 81 +++- pyproject.toml | 1 + tests/test_scope_import.py | 312 +++++++++++++ uv.lock | 23 + 18 files changed, 1165 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/d1e2f3a4b5c6_add_scope_infos_table.py create mode 100644 app/api/scope_import.py create mode 100644 app/crud/scope_info.py create mode 100644 app/models/scope_info.py create mode 100644 app/schemas/scope_import.py create mode 100644 app/schemas/scope_info.py create mode 100644 app/services/scope_import.py create mode 100644 app/templates/admin/_scope_import_result.html create mode 100644 app/templates/admin/scope_import.html create mode 100644 tests/test_scope_import.py diff --git a/alembic/versions/d1e2f3a4b5c6_add_scope_infos_table.py b/alembic/versions/d1e2f3a4b5c6_add_scope_infos_table.py new file mode 100644 index 0000000..b0f4e61 --- /dev/null +++ b/alembic/versions/d1e2f3a4b5c6_add_scope_infos_table.py @@ -0,0 +1,38 @@ +"""add_scope_infos_table + +Revision ID: d1e2f3a4b5c6 +Revises: c9f1a2b3d4e5 +Create Date: 2026-07-16 20:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "d1e2f3a4b5c6" +down_revision: Union[str, None] = "c9f1a2b3d4e5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scope_infos", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("augmented_part_id", sa.Integer(), nullable=False), + sa.Column("magnification_min", sa.Float(), nullable=True), + sa.Column("magnification_max", sa.Float(), nullable=True), + sa.Column("model_suffix", sa.String(), nullable=True), + sa.Column("objective_thread_pitch", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["augmented_part_id"], ["augmented_parts.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("augmented_part_id"), + ) + + +def downgrade() -> None: + op.drop_table("scope_infos") diff --git a/app/api/router.py b/app/api/router.py index bc5a2af..9773f94 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -14,6 +14,7 @@ from app.api import ( part_colours, part_types, roles, + scope_import, scope_models, static_connections, sub_part_types, @@ -66,3 +67,6 @@ api_router.include_router( ) api_router.include_router(audit_log.router, prefix="/audit-log", tags=["audit-log"]) api_router.include_router(fits.router, prefix="/fits", tags=["fits"]) +api_router.include_router( + scope_import.router, prefix="/scope-import", tags=["scope-import"] +) diff --git a/app/api/scope_import.py b/app/api/scope_import.py new file mode 100644 index 0000000..9c92bef --- /dev/null +++ b/app/api/scope_import.py @@ -0,0 +1,53 @@ +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.crud import audit as audit_crud +from app.deps import require_permission +from app.models.user import User +from app.schemas.scope_import import ScopeImportResult +from app.services import scope_import as scope_import_service + +router = APIRouter() + +_XLSX_TYPES = { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/octet-stream", + "", +} + + +@router.post("", response_model=ScopeImportResult) +async def import_scopes( + file: UploadFile = File(...), + db: AsyncSession = Depends(get_db), + actor: User = Depends(require_permission("augmented_parts:write")), +): + filename = file.filename or "" + if not filename.lower().endswith(".xlsx"): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Expected an .xlsx spreadsheet.", + ) + data = await file.read() + try: + result = await scope_import_service.import_from_bytes(db, data) + except Exception as exc: # openpyxl / malformed workbook + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Could not read spreadsheet: {exc}", + ) from exc + + await audit_crud.log( + db, + actor=actor, + action="update", + resource_type="AugmentedPart", + resource_id=None, + description=( + f"Scope import ({filename}): {result.created} created, " + f"{result.updated} updated, {result.skipped} skipped, " + f"{result.errors} errors" + ), + ) + return result diff --git a/app/crud/augmented_part.py b/app/crud/augmented_part.py index 9fee5f7..7e5f3e8 100644 --- a/app/crud/augmented_part.py +++ b/app/crud/augmented_part.py @@ -12,6 +12,13 @@ async def get(db: AsyncSession, augmented_part_id: int) -> AugmentedPart | None: return await db.get(AugmentedPart, augmented_part_id) +async def get_by_sku(db: AsyncSession, sku: str) -> AugmentedPart | None: + result = await db.execute( + select(AugmentedPart).where(AugmentedPart.sku == sku).order_by(AugmentedPart.id) + ) + return result.scalars().first() + + async def list_all( db: AsyncSession, skip: int = 0, limit: int = 100 ) -> list[AugmentedPart]: diff --git a/app/crud/scope_info.py b/app/crud/scope_info.py new file mode 100644 index 0000000..2677143 --- /dev/null +++ b/app/crud/scope_info.py @@ -0,0 +1,56 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.scope_info import ScopeInfo + + +async def get(db: AsyncSession, scope_info_id: int) -> ScopeInfo | None: + return await db.get(ScopeInfo, scope_info_id) + + +async def get_by_augmented_part( + db: AsyncSession, augmented_part_id: int +) -> ScopeInfo | None: + result = await db.execute( + select(ScopeInfo).where(ScopeInfo.augmented_part_id == augmented_part_id) + ) + return result.scalar_one_or_none() + + +async def create( + db: AsyncSession, + *, + augmented_part_id: int, + magnification_min: float | None = None, + magnification_max: float | None = None, + model_suffix: str | None = None, + objective_thread_pitch: str | None = None, +) -> ScopeInfo: + info = ScopeInfo( + augmented_part_id=augmented_part_id, + magnification_min=magnification_min, + magnification_max=magnification_max, + model_suffix=model_suffix, + objective_thread_pitch=objective_thread_pitch, + ) + db.add(info) + await db.commit() + await db.refresh(info) + return info + + +async def upsert(db: AsyncSession, *, augmented_part_id: int, **fields) -> ScopeInfo: + """Create the ScopeInfo for an AugmentedPart, or update it if it exists.""" + info = await get_by_augmented_part(db, augmented_part_id) + if info is None: + return await create(db, augmented_part_id=augmented_part_id, **fields) + for field, value in fields.items(): + setattr(info, field, value) + await db.commit() + await db.refresh(info) + return info + + +async def delete(db: AsyncSession, info: ScopeInfo) -> None: + await db.delete(info) + await db.commit() diff --git a/app/crud/scope_model.py b/app/crud/scope_model.py index 6c82f2b..6283d20 100644 --- a/app/crud/scope_model.py +++ b/app/crud/scope_model.py @@ -8,6 +8,18 @@ async def get(db: AsyncSession, scope_model_id: int) -> ScopeModel | None: return await db.get(ScopeModel, scope_model_id) +async def get_by_name_and_manufacturer( + db: AsyncSession, name: str, manufacturer_id: int +) -> ScopeModel | None: + result = await db.execute( + select(ScopeModel).where( + ScopeModel.name == name, + ScopeModel.manufacturer_id == manufacturer_id, + ) + ) + return result.scalar_one_or_none() + + async def list_all( db: AsyncSession, skip: int = 0, limit: int = 100 ) -> list[ScopeModel]: diff --git a/app/models/__init__.py b/app/models/__init__.py index 9870033..67d0248 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -18,6 +18,7 @@ from app.models.manufacturer import Manufacturer from app.models.part_colour import PartColour from app.models.part_type import PartType from app.models.permission import Permission +from app.models.scope_info import ScopeInfo from app.models.scope_model import ScopeModel from app.models.static_connection import StaticConnection from app.models.sub_part_type import SubPartType @@ -42,6 +43,7 @@ __all__ = [ "PartColour", "PartType", "Permission", + "ScopeInfo", "ScopeModel", "StaticConnection", "SubPartType", diff --git a/app/models/scope_info.py b/app/models/scope_info.py new file mode 100644 index 0000000..abc9d29 --- /dev/null +++ b/app/models/scope_info.py @@ -0,0 +1,34 @@ +from sqlalchemy import Float, ForeignKey, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.database import Base + + +class ScopeInfo(Base): + """Scope-specific attributes for an AugmentedPart of PartType "Scope". + + One-to-one with AugmentedPart (unique augmented_part_id). Holds fields that + have no home on AugmentedPart itself; populated by the scope xlsx importer. + """ + + __tablename__ = "scope_infos" + __table_args__ = (UniqueConstraint("augmented_part_id"),) + + id: Mapped[int] = mapped_column(primary_key=True) + augmented_part_id: Mapped[int] = mapped_column( + ForeignKey("augmented_parts.id", ondelete="CASCADE"), nullable=False + ) + magnification_min: Mapped[float | None] = mapped_column( + Float, nullable=True, default=None + ) + magnification_max: Mapped[float | None] = mapped_column( + Float, nullable=True, default=None + ) + model_suffix: Mapped[str | None] = mapped_column( + String, nullable=True, default=None + ) + # Kept as the raw string from the sheet (e.g. "0.75", "1/16") so no precision + # or notation is lost; the parsed thread lives on its own AttachmentPoint. + objective_thread_pitch: Mapped[str | None] = mapped_column( + String, nullable=True, default=None + ) diff --git a/app/schemas/scope_import.py b/app/schemas/scope_import.py new file mode 100644 index 0000000..01deeaf --- /dev/null +++ b/app/schemas/scope_import.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel + + +class ScopeImportRowResult(BaseModel): + row: int + product_number: str | None = None + status: str # "created" | "updated" | "skipped" | "error" + messages: list[str] = [] + + +class ScopeImportResult(BaseModel): + created: int + updated: int + skipped: int + errors: int + total: int + rows: list[ScopeImportRowResult] = [] diff --git a/app/schemas/scope_info.py b/app/schemas/scope_info.py new file mode 100644 index 0000000..3b6cfa2 --- /dev/null +++ b/app/schemas/scope_info.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, ConfigDict + + +class ScopeInfoBase(BaseModel): + magnification_min: float | None = None + magnification_max: float | None = None + model_suffix: str | None = None + objective_thread_pitch: str | None = None + + +class ScopeInfoRead(ScopeInfoBase): + model_config = ConfigDict(from_attributes=True) + + id: int + augmented_part_id: int diff --git a/app/services/scope_import.py b/app/services/scope_import.py new file mode 100644 index 0000000..26a0e33 --- /dev/null +++ b/app/services/scope_import.py @@ -0,0 +1,427 @@ +"""Import third-party scopes from an xlsx spreadsheet. + +The spreadsheet describes parts of PartType "Scope". Each row maps onto an +AugmentedPart (keyed on ``product_number`` → ``sku``, so re-importing upserts), +a ScopeModel, a ScopeInfo record for the scope-specific fields, up to three +attachment points, and up to two Fits against other parts. + +Column → destination +-------------------- +manufacturer → Manufacturer (get-or-create) → AugmentedPart.manufacturer +model → ScopeModel.name (get-or-create) → AugmentedPart.scope_model +url → ScopeModel.url +model_suffix → ScopeInfo.model_suffix (+ description) +sub_part_type → SubPartType (get-or-create under "Scope") +product_number → AugmentedPart.sku (also the upsert / fit-lookup key) +magnification_min / _max → ScopeInfo +objective_thread_pitch → ScopeInfo +ocular_outer_diameter_mm → StaticConnection + AttachmentPoint "Ocular" +objective_lens_outside_diameter_mm → StaticConnection + AttachmentPoint "Objective" +objective_thread → ISO/UNC thread + AttachmentPoint "Objective" +ocular_fit → Fit: scope's Ocular AP ↔ referenced part's first AP +objective_fit → Fit: scope's Objective AP ↔ referenced part's first AP + +AugmentedPart.description is synthesised from "{manufacturer} {model} {suffix}" +and colour defaults to an "Unknown" PartColour, since the sheet carries neither. + +Fits are created with label POSSIBLE. The referenced part is looked up by +``product_number`` and linked to its first attachment point (chosen answer: +"sole/first AP"). Both directions are resolved in a second pass so a fit between +two scopes in the same spreadsheet resolves regardless of row order. + +Bad rows (missing key, unknown fit target, unparseable thread) are skipped and +reported per-row; good rows are committed as they go. +""" + +import io + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.crud import attachment_point as attachment_point_crud +from app.crud import attachment_point_name as apn_crud +from app.crud import augmented_part as augmented_part_crud +from app.crud import fit as fit_crud +from app.crud import iso_thread as iso_thread_crud +from app.crud import manufacturer as manufacturer_crud +from app.crud import part_colour as part_colour_crud +from app.crud import part_type as part_type_crud +from app.crud import scope_info as scope_info_crud +from app.crud import scope_model as scope_model_crud +from app.crud import static_connection as static_connection_crud +from app.crud import sub_part_type as sub_part_type_crud +from app.crud import unc_thread as unc_thread_crud +from app.models.fit import FitLabel +from app.schemas.scope_import import ScopeImportResult, ScopeImportRowResult +from app.services.augmented_part_sync import ( + MM_PER_INCH, + _parse_thread_attribute, +) + +PART_TYPE_NAME = "Scope" +DEFAULT_COLOUR = "Unknown" +OCULAR_LABEL = "Ocular" +OBJECTIVE_LABEL = "Objective" + +# Columns we read from the sheet (normalised: stripped + lower-cased headers). +EXPECTED_COLUMNS = [ + "manufacturer", + "model", + "magnification_min", + "magnification_max", + "objective_lens_outside_diameter_mm", + "model_suffix", + "sub_part_type", + "product_number", + "url", + "ocular_outer_diameter_mm", + "ocular_fit", + "objective_thread_pitch", + "objective_thread", + "objective_fit", +] + + +def _text(value) -> str | None: + """Coerce a spreadsheet cell to a trimmed string (None if empty).""" + if value is None: + return None + # openpyxl surfaces integer-valued cells as floats (12345.0); render cleanly + # so product numbers/suffixes don't gain a stray ".0". + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + s = str(value).strip() + return s or None + + +def _num(value) -> float | None: + """Coerce a spreadsheet cell to a float (None if empty/non-numeric).""" + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return None + try: + return float(s) + except ValueError: + return None + + +def parse_workbook(data: bytes) -> list[dict]: + """Read the first worksheet into a list of {column_name: cell} dicts. + + The header row is matched case-insensitively; unknown columns are ignored + and fully-blank rows are dropped. + """ + from openpyxl import load_workbook + + wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True) + try: + ws = wb.active + rows = ws.iter_rows(values_only=True) + try: + header = next(rows) + except StopIteration: + return [] + headers = [(_text(h) or "").lower() for h in header] + + records: list[dict] = [] + for raw in rows: + if all(c is None or (isinstance(c, str) and not c.strip()) for c in raw): + continue + record = {} + for idx, name in enumerate(headers): + if name: + record[name] = raw[idx] if idx < len(raw) else None + records.append(record) + return records + finally: + wb.close() + + +async def _static_ap( + db: AsyncSession, *, augmented_part_id: int, label: str, od_mm: float +): + """Create an attachment point carrying a StaticConnection for ``od_mm``.""" + sc = await static_connection_crud.get_by_od_mm(db, od_mm) + if sc is None: + sc = await static_connection_crud.create( + db, od_mm=od_mm, od_in=od_mm / MM_PER_INCH + ) + apn = await apn_crud.get_by_name(db, label) + if apn is None: + apn = await apn_crud.create(db, name=label) + return await attachment_point_crud.create( + db, + augmented_part_id=augmented_part_id, + attachment_point_name_id=apn.id, + static_connection_id=sc.id, + ) + + +async def _thread_ap( + db: AsyncSession, *, augmented_part_id: int, label: str, spec: str +): + """Create an attachment point for a parsed ISO/UNC thread, or None.""" + parsed = _parse_thread_attribute(spec) + if parsed is None: + return None + kind, fields = parsed + apn = await apn_crud.get_by_name(db, label) + if apn is None: + apn = await apn_crud.create(db, name=label) + + if kind == "iso": + thread = await iso_thread_crud.get_by_values(db, depth_mm=None, **fields) + if thread is None: + thread = await iso_thread_crud.create(db, depth_mm=None, **fields) + return await attachment_point_crud.create( + db, + augmented_part_id=augmented_part_id, + attachment_point_name_id=apn.id, + iso_thread_id=thread.id, + ) + # unc + thread = await unc_thread_crud.get_by_values(db, depth_in=None, **fields) + if thread is None: + thread = await unc_thread_crud.create(db, depth_in=None, **fields) + return await attachment_point_crud.create( + db, + augmented_part_id=augmented_part_id, + attachment_point_name_id=apn.id, + unc_thread_id=thread.id, + ) + + +async def _link_fit( + db: AsyncSession, + *, + scope_ap, + scope_role: str, + target_sku: str, + scope_part_id: int, + messages: list[str], +) -> None: + """Create a POSSIBLE fit from the scope's AP to the referenced part's first AP.""" + if scope_ap is None: + messages.append( + f"{scope_role}_fit '{target_sku}' skipped: scope has no " + f"{scope_role} attachment point" + ) + return + target = await augmented_part_crud.get_by_sku(db, target_sku) + if target is None: + messages.append(f"{scope_role}_fit references unknown product '{target_sku}'") + return + if target.id == scope_part_id: + messages.append(f"{scope_role}_fit '{target_sku}' skipped: self-reference") + return + target_aps = await attachment_point_crud.list_for_augmented_part(db, target.id) + if not target_aps: + messages.append( + f"{scope_role}_fit '{target_sku}' skipped: referenced part has no " + "attachment points" + ) + return + target_ap = target_aps[0] + existing = await fit_crud.get_by_attachment_points(db, scope_ap, target_ap) + if existing is None: + await fit_crud.create( + db, ap_a=scope_ap, ap_b=target_ap, label=FitLabel.POSSIBLE + ) + + +async def import_scopes(db: AsyncSession, rows: list[dict]) -> ScopeImportResult: + # Shared lookups created once up front. + part_type = await part_type_crud.get_by_name(db, PART_TYPE_NAME) + if part_type is None: + part_type = await part_type_crud.create(db, name=PART_TYPE_NAME) + colour = await part_colour_crud.get_by_name(db, DEFAULT_COLOUR) + if colour is None: + colour = await part_colour_crud.create(db, name=DEFAULT_COLOUR) + + created = updated = skipped = errors = 0 + row_results: list[ScopeImportRowResult] = [] + # (row_dict, scope_ap_ocular, scope_ap_objective, part_id, row_result) + staged: list[tuple] = [] + + # ---- Phase 1: parts, scope info, attachment points ---------------------- + for offset, row in enumerate(rows): + # +2: 1-based, and the header occupies row 1. + row_num = offset + 2 + product_number = _text(row.get("product_number")) or _text(row.get("model")) + result = ScopeImportRowResult( + row=row_num, product_number=product_number, status="skipped", messages=[] + ) + + try: + if not product_number: + result.messages.append("missing product_number") + skipped += 1 + row_results.append(result) + continue + + manufacturer_name = _text(row.get("manufacturer")) + if not manufacturer_name: + result.messages.append("missing manufacturer") + skipped += 1 + row_results.append(result) + continue + + manufacturer = await manufacturer_crud.get_by_name(db, manufacturer_name) + if manufacturer is None: + manufacturer = await manufacturer_crud.create( + db, name=manufacturer_name, url="" + ) + + model_name = _text(row.get("model")) + url = _text(row.get("url")) + scope_model = None + if model_name: + scope_model = await scope_model_crud.get_by_name_and_manufacturer( + db, model_name, manufacturer.id + ) + if scope_model is None: + scope_model = await scope_model_crud.create( + db, name=model_name, manufacturer_id=manufacturer.id, url=url + ) + + sub_part_type_name = _text(row.get("sub_part_type")) + sub_part_type = None + if sub_part_type_name: + sub_part_type = await sub_part_type_crud.get_by_name_and_part_type( + db, sub_part_type_name, part_type.id + ) + if sub_part_type is None: + sub_part_type = await sub_part_type_crud.create( + db, name=sub_part_type_name, part_type_id=part_type.id + ) + + model_suffix = _text(row.get("model_suffix")) + description = ( + " ".join(p for p in (manufacturer_name, model_name, model_suffix) if p) + or product_number + ) + + fields = dict( + sku=product_number, + description=description, + colour_id=colour.id, + manufacturer_id=manufacturer.id, + scope_model_id=scope_model.id if scope_model else None, + part_type_id=part_type.id, + sub_part_type_id=sub_part_type.id if sub_part_type else None, + ) + + existing = await augmented_part_crud.get_by_sku(db, product_number) + if existing is not None: + part = await augmented_part_crud.update(db, existing, fields) + result.status = "updated" + updated += 1 + else: + part = await augmented_part_crud.create(db, **fields) + result.status = "created" + created += 1 + + await scope_info_crud.upsert( + db, + augmented_part_id=part.id, + magnification_min=_num(row.get("magnification_min")), + magnification_max=_num(row.get("magnification_max")), + model_suffix=model_suffix, + objective_thread_pitch=_text(row.get("objective_thread_pitch")), + ) + + # Rebuild attachment points from scratch so re-imports stay idempotent + # (cascades delete any fits that referenced the old points). + for old_ap in await attachment_point_crud.list_for_augmented_part( + db, part.id + ): + await attachment_point_crud.delete(db, old_ap) + + ocular_ap = None + ocular_od = _num(row.get("ocular_outer_diameter_mm")) + if ocular_od is not None: + ocular_ap = await _static_ap( + db, + augmented_part_id=part.id, + label=OCULAR_LABEL, + od_mm=ocular_od, + ) + + objective_static_ap = None + objective_od = _num(row.get("objective_lens_outside_diameter_mm")) + if objective_od is not None: + objective_static_ap = await _static_ap( + db, + augmented_part_id=part.id, + label=OBJECTIVE_LABEL, + od_mm=objective_od, + ) + + objective_thread_spec = _text(row.get("objective_thread")) + objective_thread_ap = None + if objective_thread_spec: + objective_thread_ap = await _thread_ap( + db, + augmented_part_id=part.id, + label=OBJECTIVE_LABEL, + spec=objective_thread_spec, + ) + if objective_thread_ap is None: + result.messages.append( + f"could not parse objective_thread " + f"'{objective_thread_spec}'" + ) + + # For objective_fit prefer the lens-OD point, falling back to the thread. + objective_ap = objective_static_ap or objective_thread_ap + + staged.append((row, ocular_ap, objective_ap, part.id, result)) + row_results.append(result) + except Exception as exc: # noqa: BLE001 - report, don't abort the batch + result.status = "error" + result.messages.append(f"unexpected error: {exc}") + errors += 1 + row_results.append(result) + + # ---- Phase 2: fits (all parts from this import now exist) ---------------- + for row, ocular_ap, objective_ap, part_id, result in staged: + try: + ocular_fit = _text(row.get("ocular_fit")) + if ocular_fit: + await _link_fit( + db, + scope_ap=ocular_ap, + scope_role=OCULAR_LABEL.lower(), + target_sku=ocular_fit, + scope_part_id=part_id, + messages=result.messages, + ) + objective_fit = _text(row.get("objective_fit")) + if objective_fit: + await _link_fit( + db, + scope_ap=objective_ap, + scope_role=OBJECTIVE_LABEL.lower(), + target_sku=objective_fit, + scope_part_id=part_id, + messages=result.messages, + ) + except Exception as exc: # noqa: BLE001 + result.messages.append(f"fit resolution error: {exc}") + + return ScopeImportResult( + created=created, + updated=updated, + skipped=skipped, + errors=errors, + total=len(rows), + rows=row_results, + ) + + +async def import_from_bytes(db: AsyncSession, data: bytes) -> ScopeImportResult: + rows = parse_workbook(data) + return await import_scopes(db, rows) diff --git a/app/templates/_nav.html b/app/templates/_nav.html index 0bbe045..037e296 100644 --- a/app/templates/_nav.html +++ b/app/templates/_nav.html @@ -112,6 +112,11 @@ + + + + + {# ── Nav macro for DRY icon use ────────────────────────────────────────────── #} @@ -206,6 +211,9 @@ {% if su or "augmented_parts:write" in p %} {{ icon("alert-circle") }} Skipped Parts {% endif %} + {% if su or "augmented_parts:write" in p %} + {{ icon("upload") }} Scope Import + {% endif %} {% endif %} diff --git a/app/templates/admin/_scope_import_result.html b/app/templates/admin/_scope_import_result.html new file mode 100644 index 0000000..4f7d858 --- /dev/null +++ b/app/templates/admin/_scope_import_result.html @@ -0,0 +1,41 @@ +{% if error %} +
+

{{ error }}

+
+{% elif result %} +
+

Import result

+

+ {{ result.created }} created, + {{ result.updated }} updated, + {{ result.skipped }} skipped, + {{ result.errors }} errors + ({{ result.total }} rows). +

+ + {% set flagged = result.rows | selectattr("messages") | list %} + {% if flagged %} + + + + + + {% for r in flagged %} + + + + + + + {% endfor %} + +
RowProduct #StatusMessages
{{ r.row }}{{ r.product_number or "—" }}{{ r.status }} +
    + {% for m in r.messages %}
  • {{ m }}
    • {% endfor %} +
+
+ {% else %} +

All rows imported cleanly.

+ {% endif %} +
+{% endif %} diff --git a/app/templates/admin/scope_import.html b/app/templates/admin/scope_import.html new file mode 100644 index 0000000..7367d2a --- /dev/null +++ b/app/templates/admin/scope_import.html @@ -0,0 +1,37 @@ +{% extends "base.html" %} +{% block title %}Scope Import · {{ app_name }}{% endblock %} +{% block body %} +{% include "_nav.html" %} +
+

Scope Import

+ +
+

Upload spreadsheet

+

+ Upload an .xlsx file describing scopes (PartType = Scope). Rows are + matched on product_number; re-uploading updates existing scopes. +

+
+ + + Importing… +
+
+ Expected columns +
    + {% for col in expected_columns %} +
  • {{ col }}
    • + {% endfor %} +
+
+
+ +
+
+{% endblock %} diff --git a/app/web/admin.py b/app/web/admin.py index 416a5f0..f636cc9 100644 --- a/app/web/admin.py +++ b/app/web/admin.py @@ -3,7 +3,16 @@ import math import urllib.parse -from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from fastapi import ( + APIRouter, + Depends, + File, + Form, + HTTPException, + Request, + UploadFile, + status, +) from fastapi.responses import HTMLResponse, RedirectResponse, Response from sqlalchemy.ext.asyncio import AsyncSession @@ -34,6 +43,7 @@ from app.models.user import User from app.services import augmented_part_sync as augmented_part_sync_service from app.services import fit_matching as fit_matching_service from app.services import odata as odata_service +from app.services import scope_import as scope_import_service from app.web.templates import templates router = APIRouter(prefix="/admin", include_in_schema=False) @@ -1344,6 +1354,64 @@ async def filter_imported_parts( ) +# --- Scope Import ------------------------------------------------------------ + + +@router.get("/scope-import", response_class=HTMLResponse) +async def scope_import_page( + request: Request, + db: AsyncSession = Depends(get_db), + user: User = Depends(web_require("augmented_parts:write")), +): + return templates.TemplateResponse( + request, + "admin/scope_import.html", + {"user": user, "expected_columns": scope_import_service.EXPECTED_COLUMNS}, + ) + + +@router.post("/scope-import", response_class=HTMLResponse) +async def upload_scope_import( + request: Request, + file: UploadFile = File(...), + db: AsyncSession = Depends(get_db), + actor: User = Depends(web_require("augmented_parts:write")), +): + filename = file.filename or "" + if not filename.lower().endswith(".xlsx"): + return templates.TemplateResponse( + request, + "admin/_scope_import_result.html", + {"error": "Expected an .xlsx spreadsheet."}, + ) + data = await file.read() + try: + result = await scope_import_service.import_from_bytes(db, data) + except Exception as exc: # noqa: BLE001 + return templates.TemplateResponse( + request, + "admin/_scope_import_result.html", + {"error": f"Could not read spreadsheet: {exc}"}, + ) + await audit_crud.log( + db, + actor=actor, + action="update", + resource_type="AugmentedPart", + resource_id=None, + description=( + f"Scope import ({filename}): {result.created} created, " + f"{result.updated} updated, {result.skipped} skipped, " + f"{result.errors} errors" + ), + ) + return templates.TemplateResponse( + request, + "admin/_scope_import_result.html", + {"result": result}, + ) + + # --- Augmented Parts --------------------------------------------------------- @@ -2026,7 +2094,12 @@ async def _fit_view(db: AsyncSession, fit, part_id: int) -> dict: fit.augmented_part_a_id, ) other_part = await augmented_part_crud.get(db, other_part_id) - return {"fit": fit, "own_ap": own_ap, "other_ap": other_ap, "other_part": other_part} + return { + "fit": fit, + "own_ap": own_ap, + "other_ap": other_ap, + "other_part": other_part, + } async def _augmented_part_fits_ctx( @@ -2224,7 +2297,9 @@ async def delete_augmented_part_fit( ): fit = await fit_crud.get(db, fit_id) if fit is not None: - label = f"attachment points {fit.attachment_point_a_id}↔{fit.attachment_point_b_id}" + label = ( + f"attachment points {fit.attachment_point_a_id}↔{fit.attachment_point_b_id}" + ) await fit_crud.delete(db, fit) await audit_crud.log( db, diff --git a/pyproject.toml b/pyproject.toml index beded91..4a7abdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "bcrypt>=4.1", "httpx>=0.27", "ldap3>=2.9", + "openpyxl>=3.1", ] [project.optional-dependencies] diff --git a/tests/test_scope_import.py b/tests/test_scope_import.py new file mode 100644 index 0000000..e4a1438 --- /dev/null +++ b/tests/test_scope_import.py @@ -0,0 +1,312 @@ +import io + +from openpyxl import Workbook + +from app.crud import attachment_point as attachment_point_crud +from app.crud import attachment_point_name as apn_crud +from app.crud import augmented_part as augmented_part_crud +from app.crud import fit as fit_crud +from app.crud import manufacturer as manufacturer_crud +from app.crud import part_colour as part_colour_crud +from app.crud import part_type as part_type_crud +from app.crud import scope_info as scope_info_crud +from app.crud import static_connection as static_connection_crud +from app.models.fit import FitLabel +from app.services import scope_import as scope_import_service + +HEADERS = [ + "manufacturer", + "model", + "magnification_min", + "magnification_max", + "objective_lens_outside_diameter_mm", + "model_suffix", + "sub_part_type", + "product_number", + "url", + "ocular_outer_diameter_mm", + "ocular_fit", + "objective_thread_pitch", + "objective_thread", + "objective_fit", +] + + +def _workbook(rows, headers=HEADERS): + # ``headers`` are the labels written to the header row (may be re-cased to + # exercise case-insensitive matching); values are always pulled by the + # canonical lower-case keys the row dicts use. + wb = Workbook() + ws = wb.active + ws.append(headers) + for r in rows: + ws.append([r.get(key) for key in HEADERS]) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _scope_row(**overrides): + row = { + "manufacturer": "Vortex", + "model": "Viper", + "magnification_min": 5, + "magnification_max": 25, + "objective_lens_outside_diameter_mm": 56, + "model_suffix": "PST", + "sub_part_type": "Riflescope", + "product_number": "SCOPE-A", + "url": "https://example.com/viper", + "ocular_outer_diameter_mm": 40, + "ocular_fit": None, + "objective_thread_pitch": "0.75", + "objective_thread": "M40x0.75", + "objective_fit": None, + } + row.update(overrides) + return row + + +async def _make_accessory(db, sku, od_mm): + """A minimal non-scope part with a single static attachment point.""" + colour = await part_colour_crud.get_by_name(db, "Black") + if colour is None: + colour = await part_colour_crud.create(db, name="Black") + manu = await manufacturer_crud.get_by_name(db, "Armament") + if manu is None: + manu = await manufacturer_crud.create(db, name="Armament", url="") + pt = await part_type_crud.get_by_name(db, "Flip Cover") + if pt is None: + pt = await part_type_crud.create(db, name="Flip Cover") + part = await augmented_part_crud.create( + db, + sku=sku, + description=f"Accessory {sku}", + colour_id=colour.id, + manufacturer_id=manu.id, + scope_model_id=None, + part_type_id=pt.id, + sub_part_type_id=None, + ) + sc = await static_connection_crud.create(db, od_mm=od_mm, od_in=od_mm / 25.4) + apn = await apn_crud.get_by_name(db, "Cover") + if apn is None: + apn = await apn_crud.create(db, name="Cover") + ap = await attachment_point_crud.create( + db, + augmented_part_id=part.id, + attachment_point_name_id=apn.id, + static_connection_id=sc.id, + ) + return part, ap + + +# ---- parsing --------------------------------------------------------------- + + +def test_parse_workbook_is_case_insensitive_and_skips_blank_rows(): + headers = [h.upper() for h in HEADERS] + data = _workbook([_scope_row(), _scope_row(product_number=None)], headers=headers) + # add a fully blank row is implicit via None product; both rows are non-blank here + rows = scope_import_service.parse_workbook(data) + assert len(rows) == 2 + assert rows[0]["manufacturer"] == "Vortex" + assert rows[0]["product_number"] == "SCOPE-A" + + +# ---- import ---------------------------------------------------------------- + + +async def test_import_creates_scope_with_info_and_attachment_points(db_sessionmaker): + async with db_sessionmaker() as db: + data = _workbook([_scope_row()]) + result = await scope_import_service.import_from_bytes(db, data) + + assert result.created == 1 + assert result.updated == 0 + assert result.total == 1 + + part = await augmented_part_crud.get_by_sku(db, "SCOPE-A") + assert part is not None + assert part.description == "Vortex Viper PST" + assert part.part_type.name == "Scope" + assert part.colour.name == "Unknown" + assert part.manufacturer.name == "Vortex" + assert part.scope_model is not None + assert part.scope_model.name == "Viper" + assert part.sub_part_type is not None + assert part.sub_part_type.name == "Riflescope" + + info = await scope_info_crud.get_by_augmented_part(db, part.id) + assert info.magnification_min == 5 + assert info.magnification_max == 25 + assert info.model_suffix == "PST" + assert info.objective_thread_pitch == "0.75" + + aps = await attachment_point_crud.list_for_augmented_part(db, part.id) + by_label = {} + for ap in aps: + by_label.setdefault(ap.attachment_point_name.name, []).append(ap) + + assert len(by_label["Ocular"]) == 1 + assert by_label["Ocular"][0].static_connection.od_mm == 40 + + objectives = by_label["Objective"] + assert len(objectives) == 2 # static OD + thread + static_obj = [a for a in objectives if a.static_connection is not None] + thread_obj = [a for a in objectives if a.iso_thread is not None] + assert static_obj and static_obj[0].static_connection.od_mm == 56 + assert thread_obj + assert thread_obj[0].iso_thread.diameter_mm == 40 + assert thread_obj[0].iso_thread.pitch_mm == 0.75 + + +async def test_import_resolves_ocular_and_objective_fits(db_sessionmaker): + async with db_sessionmaker() as db: + _, acc1_ap = await _make_accessory(db, "ACC-1", od_mm=40) + _, acc2_ap = await _make_accessory(db, "ACC-2", od_mm=56) + + data = _workbook([_scope_row(ocular_fit="ACC-1", objective_fit="ACC-2")]) + result = await scope_import_service.import_from_bytes(db, data) + assert result.created == 1 + + part = await augmented_part_crud.get_by_sku(db, "SCOPE-A") + fits = await fit_crud.list_for_part(db, part.id) + assert len(fits) == 2 + for f in fits: + assert f.label == FitLabel.POSSIBLE + linked_ap_ids = {f.attachment_point_a_id for f in fits} | { + f.attachment_point_b_id for f in fits + } + assert acc1_ap.id in linked_ap_ids + assert acc2_ap.id in linked_ap_ids + + +async def test_import_upserts_on_product_number(db_sessionmaker): + async with db_sessionmaker() as db: + await scope_import_service.import_from_bytes(db, _workbook([_scope_row()])) + # Re-import same product number with a changed model. + result = await scope_import_service.import_from_bytes( + db, _workbook([_scope_row(model="Razor")]) + ) + assert result.updated == 1 + assert result.created == 0 + + # Only one part with that sku, and its model was updated. + parts = await augmented_part_crud.list_all(db) + matching = [p for p in parts if p.sku == "SCOPE-A"] + assert len(matching) == 1 + assert matching[0].scope_model.name == "Razor" + + # Attachment points were rebuilt, not duplicated. + aps = await attachment_point_crud.list_for_augmented_part(db, matching[0].id) + assert len(aps) == 3 + + +async def test_row_without_product_number_is_skipped(db_sessionmaker): + async with db_sessionmaker() as db: + data = _workbook([_scope_row(product_number=None)]) + result = await scope_import_service.import_from_bytes(db, data) + assert result.created == 0 + assert result.skipped == 1 + assert result.rows[0].status == "skipped" + assert "missing product_number" in result.rows[0].messages + + +async def test_unparseable_objective_thread_is_reported(db_sessionmaker): + async with db_sessionmaker() as db: + data = _workbook([_scope_row(objective_thread="not-a-thread")]) + result = await scope_import_service.import_from_bytes(db, data) + assert result.created == 1 + msgs = " ".join(result.rows[0].messages) + assert "could not parse objective_thread" in msgs + + +async def test_unknown_fit_reference_is_reported(db_sessionmaker): + async with db_sessionmaker() as db: + data = _workbook([_scope_row(ocular_fit="DOES-NOT-EXIST")]) + result = await scope_import_service.import_from_bytes(db, data) + assert result.created == 1 + msgs = " ".join(result.rows[0].messages) + assert "unknown product 'DOES-NOT-EXIST'" in msgs + + part = await augmented_part_crud.get_by_sku(db, "SCOPE-A") + assert await fit_crud.list_for_part(db, part.id) == [] + + +async def test_unc_objective_thread_creates_unc_attachment_point(db_sessionmaker): + async with db_sessionmaker() as db: + data = _workbook([_scope_row(objective_thread="1.750-32 UNS")]) + await scope_import_service.import_from_bytes(db, data) + part = await augmented_part_crud.get_by_sku(db, "SCOPE-A") + aps = await attachment_point_crud.list_for_augmented_part(db, part.id) + unc_aps = [a for a in aps if a.unc_thread is not None] + assert len(unc_aps) == 1 + assert unc_aps[0].unc_thread.diameter_fraction == "1.750" + assert unc_aps[0].unc_thread.tpi == 32 + + +# ---- API endpoint ---------------------------------------------------------- + + +async def _token(client, email, password="secret123"): + resp = await client.post( + "/api/auth/login", data={"username": email, "password": password} + ) + assert resp.status_code == 200, resp.text + return resp.json()["access_token"] + + +async def test_api_upload_requires_permission(client, seeded, make_user): + # "user" role only grants reads, not augmented_parts:write. + await make_user("reader@example.com", role_ids=[seeded["roles"]["user"].id]) + headers = {"Authorization": f"Bearer {await _token(client, 'reader@example.com')}"} + files = { + "file": ( + "scopes.xlsx", + _workbook([_scope_row()]), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + } + resp = await client.post("/api/scope-import", files=files, headers=headers) + assert resp.status_code == 403 + + +async def test_api_upload_imports_scopes(client, seeded, make_user): + await make_user("root@example.com", is_superuser=True) + headers = {"Authorization": f"Bearer {await _token(client, 'root@example.com')}"} + files = { + "file": ( + "scopes.xlsx", + _workbook([_scope_row()]), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + } + resp = await client.post("/api/scope-import", files=files, headers=headers) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["created"] == 1 + assert body["total"] == 1 + assert body["rows"][0]["product_number"] == "SCOPE-A" + + +async def test_api_upload_rejects_non_xlsx(client, seeded, make_user): + await make_user("root2@example.com", is_superuser=True) + headers = {"Authorization": f"Bearer {await _token(client, 'root2@example.com')}"} + files = {"file": ("scopes.csv", b"not,a,spreadsheet", "text/csv")} + resp = await client.post("/api/scope-import", files=files, headers=headers) + assert resp.status_code == 422 + + +async def test_fits_between_two_scopes_in_same_sheet_resolve(db_sessionmaker): + async with db_sessionmaker() as db: + rows = [ + _scope_row(product_number="SCOPE-A", ocular_fit="SCOPE-B"), + _scope_row(product_number="SCOPE-B", model="Diamondback"), + ] + result = await scope_import_service.import_from_bytes(db, _workbook(rows)) + assert result.created == 2 + + a = await augmented_part_crud.get_by_sku(db, "SCOPE-A") + fits = await fit_crud.list_for_part(db, a.id) + assert len(fits) == 1 diff --git a/uv.lock b/uv.lock index 093735c..3869ce8 100644 --- a/uv.lock +++ b/uv.lock @@ -263,6 +263,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "fastapi" version = "0.138.0" @@ -292,6 +301,7 @@ dependencies = [ { name = "httpx" }, { name = "jinja2" }, { name = "ldap3" }, + { name = "openpyxl" }, { name = "pydantic", extra = ["email"] }, { name = "pydantic-settings" }, { name = "pyjwt" }, @@ -321,6 +331,7 @@ requires-dist = [ { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "ldap3", specifier = ">=2.9" }, + { name = "openpyxl", specifier = ">=3.1" }, { name = "pydantic", extras = ["email"], specifier = ">=2.7" }, { name = "pydantic-settings", specifier = ">=2.3" }, { name = "pyjwt", specifier = ">=2.8" }, @@ -627,6 +638,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "packaging" version = "26.2"