Adding scope import

app/services/scope_import.py

@@ -0,0 +1,427 @@
+"""Import third-party scopes from an xlsx spreadsheet.
+
+The spreadsheet describes parts of PartType "Scope". Each row maps onto an
+AugmentedPart (keyed on ``product_number`` → ``sku``, so re-importing upserts),
+a ScopeModel, a ScopeInfo record for the scope-specific fields, up to three
+attachment points, and up to two Fits against other parts.
+
+Column → destination
+--------------------
+manufacturer                        → Manufacturer (get-or-create) → AugmentedPart.manufacturer
+model                               → ScopeModel.name (get-or-create) → AugmentedPart.scope_model
+url                                 → ScopeModel.url
+model_suffix                        → ScopeInfo.model_suffix (+ description)
+sub_part_type                       → SubPartType (get-or-create under "Scope")
+product_number                      → AugmentedPart.sku  (also the upsert / fit-lookup key)
+magnification_min / _max            → ScopeInfo
+objective_thread_pitch              → ScopeInfo
+ocular_outer_diameter_mm            → StaticConnection + AttachmentPoint "Ocular"
+objective_lens_outside_diameter_mm  → StaticConnection + AttachmentPoint "Objective"
+objective_thread                    → ISO/UNC thread + AttachmentPoint "Objective"
+ocular_fit                          → Fit: scope's Ocular AP ↔ referenced part's first AP
+objective_fit                       → Fit: scope's Objective AP ↔ referenced part's first AP
+
+AugmentedPart.description is synthesised from "{manufacturer} {model} {suffix}"
+and colour defaults to an "Unknown" PartColour, since the sheet carries neither.
+
+Fits are created with label POSSIBLE. The referenced part is looked up by
+``product_number`` and linked to its first attachment point (chosen answer:
+"sole/first AP"). Both directions are resolved in a second pass so a fit between
+two scopes in the same spreadsheet resolves regardless of row order.
+
+Bad rows (missing key, unknown fit target, unparseable thread) are skipped and
+reported per-row; good rows are committed as they go.
+"""
+
+import io
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.crud import attachment_point as attachment_point_crud
+from app.crud import attachment_point_name as apn_crud
+from app.crud import augmented_part as augmented_part_crud
+from app.crud import fit as fit_crud
+from app.crud import iso_thread as iso_thread_crud
+from app.crud import manufacturer as manufacturer_crud
+from app.crud import part_colour as part_colour_crud
+from app.crud import part_type as part_type_crud
+from app.crud import scope_info as scope_info_crud
+from app.crud import scope_model as scope_model_crud
+from app.crud import static_connection as static_connection_crud
+from app.crud import sub_part_type as sub_part_type_crud
+from app.crud import unc_thread as unc_thread_crud
+from app.models.fit import FitLabel
+from app.schemas.scope_import import ScopeImportResult, ScopeImportRowResult
+from app.services.augmented_part_sync import (
+    MM_PER_INCH,
+    _parse_thread_attribute,
+)
+
+PART_TYPE_NAME = "Scope"
+DEFAULT_COLOUR = "Unknown"
+OCULAR_LABEL = "Ocular"
+OBJECTIVE_LABEL = "Objective"
+
+# Columns we read from the sheet (normalised: stripped + lower-cased headers).
+EXPECTED_COLUMNS = [
+    "manufacturer",
+    "model",
+    "magnification_min",
+    "magnification_max",
+    "objective_lens_outside_diameter_mm",
+    "model_suffix",
+    "sub_part_type",
+    "product_number",
+    "url",
+    "ocular_outer_diameter_mm",
+    "ocular_fit",
+    "objective_thread_pitch",
+    "objective_thread",
+    "objective_fit",
+]
+
+
+def _text(value) -> str | None:
+    """Coerce a spreadsheet cell to a trimmed string (None if empty)."""
+    if value is None:
+        return None
+    # openpyxl surfaces integer-valued cells as floats (12345.0); render cleanly
+    # so product numbers/suffixes don't gain a stray ".0".
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    s = str(value).strip()
+    return s or None
+
+
+def _num(value) -> float | None:
+    """Coerce a spreadsheet cell to a float (None if empty/non-numeric)."""
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    s = str(value).strip()
+    if not s:
+        return None
+    try:
+        return float(s)
+    except ValueError:
+        return None
+
+
+def parse_workbook(data: bytes) -> list[dict]:
+    """Read the first worksheet into a list of {column_name: cell} dicts.
+
+    The header row is matched case-insensitively; unknown columns are ignored
+    and fully-blank rows are dropped.
+    """
+    from openpyxl import load_workbook
+
+    wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+    try:
+        ws = wb.active
+        rows = ws.iter_rows(values_only=True)
+        try:
+            header = next(rows)
+        except StopIteration:
+            return []
+        headers = [(_text(h) or "").lower() for h in header]
+
+        records: list[dict] = []
+        for raw in rows:
+            if all(c is None or (isinstance(c, str) and not c.strip()) for c in raw):
+                continue
+            record = {}
+            for idx, name in enumerate(headers):
+                if name:
+                    record[name] = raw[idx] if idx < len(raw) else None
+            records.append(record)
+        return records
+    finally:
+        wb.close()
+
+
+async def _static_ap(
+    db: AsyncSession, *, augmented_part_id: int, label: str, od_mm: float
+):
+    """Create an attachment point carrying a StaticConnection for ``od_mm``."""
+    sc = await static_connection_crud.get_by_od_mm(db, od_mm)
+    if sc is None:
+        sc = await static_connection_crud.create(
+            db, od_mm=od_mm, od_in=od_mm / MM_PER_INCH
+        )
+    apn = await apn_crud.get_by_name(db, label)
+    if apn is None:
+        apn = await apn_crud.create(db, name=label)
+    return await attachment_point_crud.create(
+        db,
+        augmented_part_id=augmented_part_id,
+        attachment_point_name_id=apn.id,
+        static_connection_id=sc.id,
+    )
+
+
+async def _thread_ap(
+    db: AsyncSession, *, augmented_part_id: int, label: str, spec: str
+):
+    """Create an attachment point for a parsed ISO/UNC thread, or None."""
+    parsed = _parse_thread_attribute(spec)
+    if parsed is None:
+        return None
+    kind, fields = parsed
+    apn = await apn_crud.get_by_name(db, label)
+    if apn is None:
+        apn = await apn_crud.create(db, name=label)
+
+    if kind == "iso":
+        thread = await iso_thread_crud.get_by_values(db, depth_mm=None, **fields)
+        if thread is None:
+            thread = await iso_thread_crud.create(db, depth_mm=None, **fields)
+        return await attachment_point_crud.create(
+            db,
+            augmented_part_id=augmented_part_id,
+            attachment_point_name_id=apn.id,
+            iso_thread_id=thread.id,
+        )
+    # unc
+    thread = await unc_thread_crud.get_by_values(db, depth_in=None, **fields)
+    if thread is None:
+        thread = await unc_thread_crud.create(db, depth_in=None, **fields)
+    return await attachment_point_crud.create(
+        db,
+        augmented_part_id=augmented_part_id,
+        attachment_point_name_id=apn.id,
+        unc_thread_id=thread.id,
+    )
+
+
+async def _link_fit(
+    db: AsyncSession,
+    *,
+    scope_ap,
+    scope_role: str,
+    target_sku: str,
+    scope_part_id: int,
+    messages: list[str],
+) -> None:
+    """Create a POSSIBLE fit from the scope's AP to the referenced part's first AP."""
+    if scope_ap is None:
+        messages.append(
+            f"{scope_role}_fit '{target_sku}' skipped: scope has no "
+            f"{scope_role} attachment point"
+        )
+        return
+    target = await augmented_part_crud.get_by_sku(db, target_sku)
+    if target is None:
+        messages.append(f"{scope_role}_fit references unknown product '{target_sku}'")
+        return
+    if target.id == scope_part_id:
+        messages.append(f"{scope_role}_fit '{target_sku}' skipped: self-reference")
+        return
+    target_aps = await attachment_point_crud.list_for_augmented_part(db, target.id)
+    if not target_aps:
+        messages.append(
+            f"{scope_role}_fit '{target_sku}' skipped: referenced part has no "
+            "attachment points"
+        )
+        return
+    target_ap = target_aps[0]
+    existing = await fit_crud.get_by_attachment_points(db, scope_ap, target_ap)
+    if existing is None:
+        await fit_crud.create(
+            db, ap_a=scope_ap, ap_b=target_ap, label=FitLabel.POSSIBLE
+        )
+
+
+async def import_scopes(db: AsyncSession, rows: list[dict]) -> ScopeImportResult:
+    # Shared lookups created once up front.
+    part_type = await part_type_crud.get_by_name(db, PART_TYPE_NAME)
+    if part_type is None:
+        part_type = await part_type_crud.create(db, name=PART_TYPE_NAME)
+    colour = await part_colour_crud.get_by_name(db, DEFAULT_COLOUR)
+    if colour is None:
+        colour = await part_colour_crud.create(db, name=DEFAULT_COLOUR)
+
+    created = updated = skipped = errors = 0
+    row_results: list[ScopeImportRowResult] = []
+    # (row_dict, scope_ap_ocular, scope_ap_objective, part_id, row_result)
+    staged: list[tuple] = []
+
+    # ---- Phase 1: parts, scope info, attachment points ----------------------
+    for offset, row in enumerate(rows):
+        # +2: 1-based, and the header occupies row 1.
+        row_num = offset + 2
+        product_number = _text(row.get("product_number")) or _text(row.get("model"))
+        result = ScopeImportRowResult(
+            row=row_num, product_number=product_number, status="skipped", messages=[]
+        )
+
+        try:
+            if not product_number:
+                result.messages.append("missing product_number")
+                skipped += 1
+                row_results.append(result)
+                continue
+
+            manufacturer_name = _text(row.get("manufacturer"))
+            if not manufacturer_name:
+                result.messages.append("missing manufacturer")
+                skipped += 1
+                row_results.append(result)
+                continue
+
+            manufacturer = await manufacturer_crud.get_by_name(db, manufacturer_name)
+            if manufacturer is None:
+                manufacturer = await manufacturer_crud.create(
+                    db, name=manufacturer_name, url=""
+                )
+
+            model_name = _text(row.get("model"))
+            url = _text(row.get("url"))
+            scope_model = None
+            if model_name:
+                scope_model = await scope_model_crud.get_by_name_and_manufacturer(
+                    db, model_name, manufacturer.id
+                )
+                if scope_model is None:
+                    scope_model = await scope_model_crud.create(
+                        db, name=model_name, manufacturer_id=manufacturer.id, url=url
+                    )
+
+            sub_part_type_name = _text(row.get("sub_part_type"))
+            sub_part_type = None
+            if sub_part_type_name:
+                sub_part_type = await sub_part_type_crud.get_by_name_and_part_type(
+                    db, sub_part_type_name, part_type.id
+                )
+                if sub_part_type is None:
+                    sub_part_type = await sub_part_type_crud.create(
+                        db, name=sub_part_type_name, part_type_id=part_type.id
+                    )
+
+            model_suffix = _text(row.get("model_suffix"))
+            description = (
+                " ".join(p for p in (manufacturer_name, model_name, model_suffix) if p)
+                or product_number
+            )
+
+            fields = dict(
+                sku=product_number,
+                description=description,
+                colour_id=colour.id,
+                manufacturer_id=manufacturer.id,
+                scope_model_id=scope_model.id if scope_model else None,
+                part_type_id=part_type.id,
+                sub_part_type_id=sub_part_type.id if sub_part_type else None,
+            )
+
+            existing = await augmented_part_crud.get_by_sku(db, product_number)
+            if existing is not None:
+                part = await augmented_part_crud.update(db, existing, fields)
+                result.status = "updated"
+                updated += 1
+            else:
+                part = await augmented_part_crud.create(db, **fields)
+                result.status = "created"
+                created += 1
+
+            await scope_info_crud.upsert(
+                db,
+                augmented_part_id=part.id,
+                magnification_min=_num(row.get("magnification_min")),
+                magnification_max=_num(row.get("magnification_max")),
+                model_suffix=model_suffix,
+                objective_thread_pitch=_text(row.get("objective_thread_pitch")),
+            )
+
+            # Rebuild attachment points from scratch so re-imports stay idempotent
+            # (cascades delete any fits that referenced the old points).
+            for old_ap in await attachment_point_crud.list_for_augmented_part(
+                db, part.id
+            ):
+                await attachment_point_crud.delete(db, old_ap)
+
+            ocular_ap = None
+            ocular_od = _num(row.get("ocular_outer_diameter_mm"))
+            if ocular_od is not None:
+                ocular_ap = await _static_ap(
+                    db,
+                    augmented_part_id=part.id,
+                    label=OCULAR_LABEL,
+                    od_mm=ocular_od,
+                )
+
+            objective_static_ap = None
+            objective_od = _num(row.get("objective_lens_outside_diameter_mm"))
+            if objective_od is not None:
+                objective_static_ap = await _static_ap(
+                    db,
+                    augmented_part_id=part.id,
+                    label=OBJECTIVE_LABEL,
+                    od_mm=objective_od,
+                )
+
+            objective_thread_spec = _text(row.get("objective_thread"))
+            objective_thread_ap = None
+            if objective_thread_spec:
+                objective_thread_ap = await _thread_ap(
+                    db,
+                    augmented_part_id=part.id,
+                    label=OBJECTIVE_LABEL,
+                    spec=objective_thread_spec,
+                )
+                if objective_thread_ap is None:
+                    result.messages.append(
+                        f"could not parse objective_thread "
+                        f"'{objective_thread_spec}'"
+                    )
+
+            # For objective_fit prefer the lens-OD point, falling back to the thread.
+            objective_ap = objective_static_ap or objective_thread_ap
+
+            staged.append((row, ocular_ap, objective_ap, part.id, result))
+            row_results.append(result)
+        except Exception as exc:  # noqa: BLE001 - report, don't abort the batch
+            result.status = "error"
+            result.messages.append(f"unexpected error: {exc}")
+            errors += 1
+            row_results.append(result)
+
+    # ---- Phase 2: fits (all parts from this import now exist) ----------------
+    for row, ocular_ap, objective_ap, part_id, result in staged:
+        try:
+            ocular_fit = _text(row.get("ocular_fit"))
+            if ocular_fit:
+                await _link_fit(
+                    db,
+                    scope_ap=ocular_ap,
+                    scope_role=OCULAR_LABEL.lower(),
+                    target_sku=ocular_fit,
+                    scope_part_id=part_id,
+                    messages=result.messages,
+                )
+            objective_fit = _text(row.get("objective_fit"))
+            if objective_fit:
+                await _link_fit(
+                    db,
+                    scope_ap=objective_ap,
+                    scope_role=OBJECTIVE_LABEL.lower(),
+                    target_sku=objective_fit,
+                    scope_part_id=part_id,
+                    messages=result.messages,
+                )
+        except Exception as exc:  # noqa: BLE001
+            result.messages.append(f"fit resolution error: {exc}")
+
+    return ScopeImportResult(
+        created=created,
+        updated=updated,
+        skipped=skipped,
+        errors=errors,
+        total=len(rows),
+        rows=row_results,
+    )
+
+
+async def import_from_bytes(db: AsyncSession, data: bytes) -> ScopeImportResult:
+    rows = parse_workbook(data)
+    return await import_scopes(db, rows)