"""Import third-party scopes from an xlsx spreadsheet. The spreadsheet describes parts of PartType "Scope". Each row maps onto an AugmentedPart (keyed on ``product_number`` → ``sku``, so re-importing upserts), a ScopeModel, a ScopeInfo record for the scope-specific fields, up to two attachment points, and up to two Fits against other parts. Column → destination -------------------- manufacturer → Manufacturer (get-or-create) → AugmentedPart.manufacturer model → ScopeModel.name (get-or-create) → AugmentedPart.scope_model url → ScopeModel.url model_suffix → ScopeInfo.model_suffix (+ description) sub_part_type → SubPartType (get-or-create under "Scope") product_number → AugmentedPart.sku (also the upsert / fit-lookup key) magnification_min / _max → ScopeInfo objective_lens_diameter_mm → ScopeInfo (descriptive only, no attachment point) ocular_outer_diameter_mm → StaticConnection + AttachmentPoint "Ocular" objective_thread → ISO/UNC thread + AttachmentPoint "Objective" ocular_fit → Fit: scope's Ocular AP ↔ referenced part's first AP objective_fit → Fit: scope's Objective AP ↔ referenced part's first AP AugmentedPart.description is synthesised from "{manufacturer} {model} {suffix}" and colour defaults to an "Unknown" PartColour, since the sheet carries neither. Fits are created with label POSSIBLE. The referenced part is looked up by ``product_number`` and linked to its first attachment point (chosen answer: "sole/first AP"). Both directions are resolved in a second pass so a fit between two scopes in the same spreadsheet resolves regardless of row order. Bad rows (missing key, unknown fit target, unparseable thread) are skipped and reported per-row; good rows are committed as they go. """ import io from sqlalchemy.ext.asyncio import AsyncSession from app.crud import attachment_point as attachment_point_crud from app.crud import attachment_point_name as apn_crud from app.crud import augmented_part as augmented_part_crud from app.crud import fit as fit_crud from app.crud import iso_thread as iso_thread_crud from app.crud import manufacturer as manufacturer_crud from app.crud import part_colour as part_colour_crud from app.crud import part_type as part_type_crud from app.crud import scope_info as scope_info_crud from app.crud import scope_model as scope_model_crud from app.crud import static_connection as static_connection_crud from app.crud import sub_part_type as sub_part_type_crud from app.crud import unc_thread as unc_thread_crud from app.models.fit import FitLabel from app.schemas.scope_import import ScopeImportResult, ScopeImportRowResult from app.services.augmented_part_sync import ( MM_PER_INCH, _parse_thread_attribute, ) PART_TYPE_NAME = "Scope" DEFAULT_COLOUR = "Unknown" OCULAR_LABEL = "Ocular" OBJECTIVE_LABEL = "Objective" # Columns we read from the sheet (normalised: stripped + lower-cased headers). EXPECTED_COLUMNS = [ "manufacturer", "model", "magnification_min", "magnification_max", "objective_lens_diameter_mm", "model_suffix", "sub_part_type", "product_number", "url", "ocular_outer_diameter_mm", "ocular_fit", "objective_thread", "objective_fit", ] def _text(value) -> str | None: """Coerce a spreadsheet cell to a trimmed string (None if empty).""" if value is None: return None # openpyxl surfaces integer-valued cells as floats (12345.0); render cleanly # so product numbers/suffixes don't gain a stray ".0". if isinstance(value, float) and value.is_integer(): return str(int(value)) s = str(value).strip() return s or None def _num(value) -> float | None: """Coerce a spreadsheet cell to a float (None if empty/non-numeric).""" if value is None: return None if isinstance(value, (int, float)): return float(value) s = str(value).strip() if not s: return None try: return float(s) except ValueError: return None def parse_workbook(data: bytes) -> list[dict]: """Read the first worksheet into a list of {column_name: cell} dicts. The header row is matched case-insensitively; unknown columns are ignored and fully-blank rows are dropped. """ from openpyxl import load_workbook wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True) try: ws = wb.active rows = ws.iter_rows(values_only=True) try: header = next(rows) except StopIteration: return [] headers = [(_text(h) or "").lower() for h in header] records: list[dict] = [] for raw in rows: if all(c is None or (isinstance(c, str) and not c.strip()) for c in raw): continue record = {} for idx, name in enumerate(headers): if name: record[name] = raw[idx] if idx < len(raw) else None records.append(record) return records finally: wb.close() async def _static_ap(db: AsyncSession, *, augmented_part_id: int, label: str, od_mm: float): """Create an attachment point carrying a StaticConnection for ``od_mm``.""" sc = await static_connection_crud.get_by_od_mm(db, od_mm) if sc is None: sc = await static_connection_crud.create(db, od_mm=od_mm, od_in=od_mm / MM_PER_INCH) apn = await apn_crud.get_by_name(db, label) if apn is None: apn = await apn_crud.create(db, name=label) return await attachment_point_crud.create( db, augmented_part_id=augmented_part_id, attachment_point_name_id=apn.id, static_connection_id=sc.id, ) async def _thread_ap(db: AsyncSession, *, augmented_part_id: int, label: str, spec: str): """Create an attachment point for a parsed ISO/UNC thread, or None.""" parsed = _parse_thread_attribute(spec) if parsed is None: return None kind, fields = parsed apn = await apn_crud.get_by_name(db, label) if apn is None: apn = await apn_crud.create(db, name=label) if kind == "iso": thread = await iso_thread_crud.get_by_values(db, depth_mm=None, **fields) if thread is None: thread = await iso_thread_crud.create(db, depth_mm=None, **fields) return await attachment_point_crud.create( db, augmented_part_id=augmented_part_id, attachment_point_name_id=apn.id, iso_thread_id=thread.id, ) # unc thread = await unc_thread_crud.get_by_values(db, depth_in=None, **fields) if thread is None: thread = await unc_thread_crud.create(db, depth_in=None, **fields) return await attachment_point_crud.create( db, augmented_part_id=augmented_part_id, attachment_point_name_id=apn.id, unc_thread_id=thread.id, ) async def _link_fit( db: AsyncSession, *, scope_ap, scope_role: str, target_sku: str, scope_part_id: int, messages: list[str], ) -> None: """Create a POSSIBLE fit from the scope's AP to the referenced part's first AP.""" if scope_ap is None: messages.append( f"{scope_role}_fit '{target_sku}' skipped: scope has no {scope_role} attachment point" ) return target = await augmented_part_crud.get_by_sku(db, target_sku) if target is None: messages.append(f"{scope_role}_fit references unknown product '{target_sku}'") return if target.id == scope_part_id: messages.append(f"{scope_role}_fit '{target_sku}' skipped: self-reference") return target_aps = await attachment_point_crud.list_for_augmented_part(db, target.id) if not target_aps: messages.append( f"{scope_role}_fit '{target_sku}' skipped: referenced part has no attachment points" ) return target_ap = target_aps[0] existing = await fit_crud.get_by_attachment_points(db, scope_ap, target_ap) if existing is None: await fit_crud.create(db, ap_a=scope_ap, ap_b=target_ap, label=FitLabel.POSSIBLE) async def import_scopes(db: AsyncSession, rows: list[dict]) -> ScopeImportResult: # Shared lookups created once up front. part_type = await part_type_crud.get_by_name(db, PART_TYPE_NAME) if part_type is None: part_type = await part_type_crud.create(db, name=PART_TYPE_NAME) colour = await part_colour_crud.get_by_name(db, DEFAULT_COLOUR) if colour is None: colour = await part_colour_crud.create(db, name=DEFAULT_COLOUR) created = updated = skipped = errors = 0 row_results: list[ScopeImportRowResult] = [] # (row_dict, scope_ap_ocular, scope_ap_objective, part_id, row_result) staged: list[tuple] = [] # ---- Phase 1: parts, scope info, attachment points ---------------------- for offset, row in enumerate(rows): # +2: 1-based, and the header occupies row 1. row_num = offset + 2 model_suffix = _text(row.get("model_suffix")) model = _text(row.get("model")) magnification_min = _num(row.get("magnification_min")) magnification_max = _num(row.get("magnification_max")) objective_lens_diameter_mm = _num(row.get("objective_lens_diameter_mm")) or _num( row.get("objective_lens") ) # Construct the model number default_product_number = model if magnification_min: default_product_number += f" {magnification_min}-{magnification_max}" if objective_lens_diameter_mm: default_product_number += f"x{objective_lens_diameter_mm}" if model_suffix: default_product_number += " {model_suffix}" product_number = _text(row.get("product_number")) or default_product_number result = ScopeImportRowResult( row=row_num, product_number=product_number, status="skipped", messages=[] ) try: if not product_number: result.messages.append("missing product_number") skipped += 1 row_results.append(result) continue manufacturer_name = _text(row.get("manufacturer")) if not manufacturer_name: result.messages.append("missing manufacturer") skipped += 1 row_results.append(result) continue manufacturer = await manufacturer_crud.get_by_name(db, manufacturer_name) if manufacturer is None: manufacturer = await manufacturer_crud.create(db, name=manufacturer_name, url="") model_name = _text(row.get("model")) url = _text(row.get("url")) scope_model = None if model_name: scope_model = await scope_model_crud.get_by_name_and_manufacturer( db, model_name, manufacturer.id ) if scope_model is None: scope_model = await scope_model_crud.create( db, name=model_name, manufacturer_id=manufacturer.id, url=url ) sub_part_type_name = _text(row.get("sub_part_type")) sub_part_type = None if sub_part_type_name: sub_part_type = await sub_part_type_crud.get_by_name_and_part_type( db, sub_part_type_name, part_type.id ) if sub_part_type is None: sub_part_type = await sub_part_type_crud.create( db, name=sub_part_type_name, part_type_id=part_type.id ) model_suffix = _text(row.get("model_suffix")) description = ( " ".join(p for p in (manufacturer_name, model_name, model_suffix) if p) or product_number ) fields = dict( sku=product_number, description=description, colour_id=colour.id, manufacturer_id=manufacturer.id, scope_model_id=scope_model.id if scope_model else None, part_type_id=part_type.id, sub_part_type_id=sub_part_type.id if sub_part_type else None, ) existing = await augmented_part_crud.get_by_sku(db, product_number) if existing is not None: part = await augmented_part_crud.update(db, existing, fields) result.status = "updated" updated += 1 else: part = await augmented_part_crud.create(db, **fields) result.status = "created" created += 1 await scope_info_crud.upsert( db, augmented_part_id=part.id, magnification_min=_num(row.get("magnification_min")), magnification_max=_num(row.get("magnification_max")), model_suffix=model_suffix, objective_lens_diameter_mm=_num(row.get("objective_lens_diameter_mm")), ) # Rebuild attachment points from scratch so re-imports stay idempotent # (cascades delete any fits that referenced the old points). for old_ap in await attachment_point_crud.list_for_augmented_part(db, part.id): await attachment_point_crud.delete(db, old_ap) ocular_ap = None ocular_od = _num(row.get("ocular_outer_diameter_mm")) if ocular_od is not None: ocular_ap = await _static_ap( db, augmented_part_id=part.id, label=OCULAR_LABEL, od_mm=ocular_od, ) # The objective attachment point comes from the thread alone — the # lens diameter is a descriptive attribute, not a connection. objective_thread_spec = _text(row.get("objective_thread")) objective_ap = None if objective_thread_spec: objective_ap = await _thread_ap( db, augmented_part_id=part.id, label=OBJECTIVE_LABEL, spec=objective_thread_spec, ) if objective_ap is None: result.messages.append( f"could not parse objective_thread '{objective_thread_spec}'" ) staged.append((row, ocular_ap, objective_ap, part.id, result)) row_results.append(result) except Exception as exc: # noqa: BLE001 - report, don't abort the batch result.status = "error" result.messages.append(f"unexpected error: {exc}") errors += 1 row_results.append(result) # ---- Phase 2: fits (all parts from this import now exist) ---------------- for row, ocular_ap, objective_ap, part_id, result in staged: try: ocular_fit = _text(row.get("ocular_fit")) if ocular_fit: fits = ocular_fit if "/" in fits: fits = [x.strip() for x in fits.split("/")] else: fits = [fits] for ocular_fit in fits: await _link_fit( db, scope_ap=ocular_ap, scope_role=OCULAR_LABEL.lower(), target_sku=ocular_fit, scope_part_id=part_id, messages=result.messages, ) objective_fit = _text(row.get("objective_fit")) if objective_fit: fits = objective_fit if "/" in fits: fits = [x.strip() for x in fits.split("/")] else: fits = [fits] for objective_fit in fits: await _link_fit( db, scope_ap=objective_ap, scope_role=OBJECTIVE_LABEL.lower(), target_sku=objective_fit, scope_part_id=part_id, messages=result.messages, ) except Exception as exc: # noqa: BLE001 result.messages.append(f"fit resolution error: {exc}") return ScopeImportResult( created=created, updated=updated, skipped=skipped, errors=errors, total=len(rows), rows=row_results, ) async def import_from_bytes(db: AsyncSession, data: bytes) -> ScopeImportResult: rows = parse_workbook(data) return await import_scopes(db, rows)