#!/usr/bin/env python3
"""Shared Canvas LMS API utilities.

Configuration via environment variables:
    CANVAS_TOKEN     — Personal access token (required)
    CANVAS_DOMAIN    — Canvas instance domain (default: canvas.asu.edu)

Course ID is passed as a parameter to get_config() or falls back to
CANVAS_COURSE_ID env var for backward compatibility.

Generate a token at: Canvas → Account → Settings → New Access Token
"""

import logging
import os
import re
import sys
import time
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from urllib.parse import urlparse

logger = logging.getLogger(__name__)
import requests

# Lazy import of backup_manager — only resolved on first guarded delete call.
# Using a function-level import avoids circular-import issues in tests that
# mock canvas_api before backup_manager is available.
def _backup_manager():
    """Return the backup_manager module, importing it on first call."""
    import backup_manager as _bm  # noqa: PLC0415
    return _bm

# Load .env from plugin root (backward-compatible)
try:
    from dotenv import load_dotenv
    load_dotenv(os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env'))
except ImportError:
    pass  # Fall back to environment variables



def validate_course_id(course_id):
    """Validate Canvas course ID is numeric. Raises ValueError otherwise."""
    cid = str(course_id).strip()
    if not re.match(r"^\d+$", cid):
        raise ValueError(f"Invalid course_id (must be numeric): {course_id!r}")
    return cid

def get_config(course_id=None):
    """Load Canvas API configuration.

    Args:
        course_id: Optional course ID. If not provided, falls back to
                   CANVAS_COURSE_ID env var for backward compatibility.

    Returns:
        Config dict with token, domain, course_id, base_url, course_url, headers.
    """
    token = os.environ.get("CANVAS_TOKEN")
    domain = os.environ.get("CANVAS_DOMAIN", "canvas.asu.edu")

    # Explicit param takes priority, then env var fallback
    if course_id is None:
        course_id = os.environ.get("CANVAS_COURSE_ID")

    if not token:
        print("ERROR: CANVAS_TOKEN environment variable not set.")
        print("Generate a token at: Canvas → Account → Settings → New Access Token")
        print("Then: export CANVAS_TOKEN='your_token_here'")
        sys.exit(1)

    return {
        "token": token,
        "domain": domain,
        "course_id": str(course_id) if course_id else None,
        "base_url": f"https://{domain}/api/v1",
        "course_url": f"https://{domain}/api/v1/courses/{course_id}" if course_id else None,
        "headers": {"Authorization": f"Bearer {token}"},
    }


def require_course_id(config):
    """Exit with error if no course ID is available."""
    if not config["course_id"]:
        print("ERROR: No course ID provided.")
        print("Pass course_id to get_config() or set CANVAS_COURSE_ID env var.")
        print("Find your course ID from the course URL: canvas.asu.edu/courses/XXXXX")
        sys.exit(1)


def _is_canvas_response_url(url, domain):
    """Return True only for HTTPS URLs on the exact configured Canvas host."""
    parsed = urlparse(url or "")
    return (
        parsed.scheme.lower() == "https"
        and (parsed.hostname or "").lower() == str(domain or "").lower()
    )


def _validate_canvas_response_url(url, domain, context):
    """Validate a Canvas-supplied URL before attaching Canvas auth headers."""
    if _is_canvas_response_url(url, domain):
        return url
    parsed = urlparse(url or "")
    actual = f"{parsed.scheme or '<missing>'}://{parsed.hostname or '<missing>'}"
    raise ValueError(
        f"Unsafe Canvas response URL for {context}: "
        f"expected https://{domain}, got {actual}"
    )


def _sanitize_error_text(text):
    """Return a short Canvas error excerpt with common secrets/PII redacted."""
    text = re.sub(r"\s+", " ", str(text or "")).strip()
    text = re.sub(
        r"(?i)\b(?:bearer|token|access_token|authorization)\s*[:=]\s*[\"']?[^\"'\s&]+",
        "[REDACTED_TOKEN]",
        text,
    )
    text = re.sub(
        r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
        "[REDACTED_EMAIL]",
        text,
    )
    text = re.sub(r"\b[A-Za-z0-9_-]{24,}\b", "[REDACTED_TOKEN]", text)
    return text[:100]


def _response_error_summary(response):
    """Return status, reason, and a sanitized short body excerpt."""
    if response is None:
        return "no response"
    status = getattr(response, "status_code", "unknown")
    reason = getattr(response, "reason", "") or ""
    excerpt = _sanitize_error_text(getattr(response, "text", ""))
    parts = [f"HTTP {status}"]
    if reason:
        parts.append(reason)
    if excerpt:
        parts.append(excerpt)
    return " - ".join(parts)


# ============================================================
# CONNECTION POOL — module-level Session singleton
# ============================================================
_SESSION = None
# HTTP verbs that go through the Session. Used to identify write operations
# for the write-only inter-request delay.
_HTTP_VERBS = frozenset({"get", "post", "put", "delete", "patch", "head", "options"})
_WRITE_VERBS = frozenset({"post", "put", "delete", "patch"})


def _session():
    """Return (and lazily create) the module-level requests.Session singleton.

    The Session reuses the underlying TCP+TLS connection pool across calls,
    removing a handshake-per-call overhead that is particularly visible in
    GET-heavy audit workflows that may issue hundreds of sequential requests.
    """
    global _SESSION
    if _SESSION is None:
        _SESSION = requests.Session()
    return _SESSION


# ============================================================
# RATE LIMITING & RETRY
# ============================================================
MAX_RETRIES = 3
BASE_DELAY = 1.0  # seconds
INTER_REQUEST_DELAY = 0.25  # seconds between write-verb API calls


def _request_with_retry(method, url, headers, max_retries=MAX_RETRIES, **kwargs):
    """Execute an HTTP request with exponential backoff on rate limits.

    Retries on HTTP 429, 500, 502, 503, 504. Proactively slows down
    when X-Rate-Limit-Remaining drops below 10. Adds a small inter-request
    delay to avoid bursty traffic.

    POST is special-cased (see GitHub issue #209): POST creates are
    non-idempotent, so a lost response after the request reached Canvas
    must not trigger a blind retry (it could create a duplicate object).
    POST retries only a connect-phase failure (ConnectionError — the
    request was provably never sent) or HTTP 429 (rejected before
    processing, so the create never happened); it does NOT retry
    ReadTimeout or 5xx, where the create may have already committed.

    Args:
        method: requests function (requests.get, requests.post, etc.)
        url: Target URL
        headers: Authorization headers
        max_retries: Number of retry attempts
        **kwargs: Passed through to requests (json, params, data, etc.)

    Returns:
        requests.Response object
    """
    # Derive the verb from the passed callable so we can (a) dispatch through
    # the Session for connection reuse and (b) apply the write-only delay.
    # Graceful fallback: if method.__name__ is not a known HTTP verb (e.g. a
    # MagicMock passed by tests), call the original callable directly.
    verb = getattr(method, "__name__", None)
    if verb not in _HTTP_VERBS:
        verb = None  # Unknown callable — fall back to direct invocation

    resp = None
    for attempt in range(max_retries):
        try:
            kwargs.setdefault("timeout", 30)
            if verb is not None:
                resp = getattr(_session(), verb)(url, headers=headers, **kwargs)
            else:
                resp = method(url, headers=headers, **kwargs)
        except requests.RequestException as exc:
            # POST is non-idempotent: a ReadTimeout means the request WAS
            # sent, so the create may have already committed on Canvas's
            # side — a blind retry risks a duplicate object. Only retry a
            # POST when the connection was never established (True for
            # ConnectTimeout/refused, False for ReadTimeout), which proves
            # the request was never sent. Every other verb is idempotent or
            # safe to repeat, so its existing retry-on-any-RequestException
            # behavior is unchanged.
            post_retryable = verb != "post" or isinstance(
                exc, requests.exceptions.ConnectionError
            )
            if attempt < max_retries - 1 and post_retryable:
                delay = BASE_DELAY * (2 ** attempt)
                print(f"  Network error ({exc.__class__.__name__}), retrying in "
                      f"{delay:.1f}s (attempt {attempt + 1}/{max_retries})")
                time.sleep(delay)
                continue
            if not post_retryable:
                # POST short-circuit (a non-ConnectionError network failure —
                # e.g. ReadTimeout): we deliberately did NOT retry. Use a
                # distinct, accurate message — telling an operator debugging a
                # lost POST that it "failed after N attempts" would wrongly
                # imply the blind-retry behavior this guard exists to prevent.
                raise requests.RequestException(
                    f"POST {url} not retried after {exc.__class__.__name__}: "
                    f"response was lost but the request may have reached Canvas; "
                    f"not retrying a non-idempotent create to avoid a duplicate. "
                    f"Original error: {exc}"
                ) from exc
            method_name = getattr(method, "__name__", "request").upper()
            raise requests.RequestException(
                f"{method_name} {url} failed after {max_retries} attempts: {exc}"
            ) from exc

        # Proactive rate limit check (unchanged — fires on every verb)
        remaining = resp.headers.get("X-Rate-Limit-Remaining")
        if remaining is not None and float(remaining) < 10:
            time.sleep(1.0)

        # Success — delay only for write verbs (POST/PUT/DELETE/PATCH).
        # GET-heavy audit workflows previously paid 0.25 s/call; they no longer
        # do. The proactive low-remaining throttle + 429 exponential backoff
        # (unchanged below) are the safety net against burst 429s on GET paths.
        if resp.status_code in (200, 201):
            if verb in _WRITE_VERBS:
                time.sleep(INTER_REQUEST_DELAY)
            return resp

        # Retryable errors — exponential backoff (honor Retry-After if present).
        # POST is non-idempotent: on a 5xx, Canvas may have already committed
        # the create even though it reported a server error, so POST does
        # NOT retry 500/502/503/504 — the caller gets the response back to
        # verify instead of risking a duplicate. 429 stays retryable for
        # POST (and every other verb): it is a pre-processing rate-limit
        # rejection, so the create never reached Canvas.
        retryable_statuses = (429,) if verb == "post" else (429, 500, 502, 503, 504)
        if resp.status_code in retryable_statuses:
            if attempt < max_retries - 1:
                ra = resp.headers.get("Retry-After")
                delay = None
                if ra is not None:
                    try:
                        delay = float(ra)
                    except ValueError:
                        try:
                            dt = parsedate_to_datetime(ra)
                            now = datetime.now(timezone.utc)
                            delay = max(0, (dt - now).total_seconds())
                        except (ValueError, TypeError):
                            delay = None
                if delay is None:
                    delay = BASE_DELAY * (2 ** attempt)
                print(f"  Rate limit/transient error (HTTP {resp.status_code}), "
                      f"retrying in {delay:.1f}s (attempt {attempt + 1}/{max_retries})")
                time.sleep(delay)
                continue

        # Non-retryable error — return immediately
        return resp

    return resp


# ============================================================
# PAGINATION
# ============================================================
# Hard ceiling on pages fetched by a single paginated_get() call. Guards
# against an unbounded loop if a Link: rel="next" header is malformed or
# self-referencing (e.g. points back at a URL already fetched).
MAX_PAGINATION_PAGES = 1000


def paginated_get(url, headers, params=None, canvas_domain=None):
    """Fetch all pages of a paginated Canvas API response."""
    results = []
    expected_domain = canvas_domain or (urlparse(url).hostname or "")
    pages_fetched = 0
    while url:
        if pages_fetched >= MAX_PAGINATION_PAGES:
            logger.warning(
                "paginated_get: hit %d-page cap fetching %s; stopping to avoid "
                "an unbounded loop (possible self-referencing next link)",
                MAX_PAGINATION_PAGES, url,
            )
            break
        resp = _request_with_retry(requests.get, url, headers, params=params)
        resp.raise_for_status()
        results.extend(resp.json())
        pages_fetched += 1
        # Parse Link header for next page. Match the URL defensively via
        # regex instead of indexing split() results — a malformed part that
        # contains rel="next" but no <...> wrapper is skipped rather than
        # raising IndexError.
        links = resp.headers.get("Link", "")
        url = None
        for part in links.split(","):
            if 'rel="next"' in part:
                match = re.search(r"<(.*?)>", part)
                if not match:
                    continue
                next_url = match.group(1)
                url = _validate_canvas_response_url(
                    next_url,
                    expected_domain,
                    "pagination next link",
                )
        params = None  # params only needed for first request
    return results


# ============================================================
# FILE UPLOAD (3-step Canvas process)
# ============================================================
def upload_file(config, filepath, folder_id, content_type="text/html", filename=None):
    """Upload a file to Canvas using the 3-step upload process.

    Args:
        config: Canvas API config dict from get_config()
        filepath: Local path to the file to upload
        folder_id: Canvas folder ID to upload into
        content_type: MIME type of the file
        filename: Optional display name to upload as. Defaults to the local
            file's basename. Pass this when the staged local filename differs
            from the Canvas display_name it must replace — e.g.
            push_remediated_file.py uploads a remediated staged copy under
            the live file's original display_name so on_duplicate=overwrite
            replaces the right Canvas file.

    Returns:
        File ID on success, None on failure
    """
    require_course_id(config)
    filename = filename or os.path.basename(filepath)
    filesize = os.path.getsize(filepath)

    # Step 1: Request upload URL
    resp = requests.post(
        f"{config['course_url']}/files",
        headers=config["headers"],
        json={
            "name": filename,
            "size": filesize,
            "content_type": content_type,
            "parent_folder_id": folder_id,
            "on_duplicate": "overwrite",
        },
        timeout=30,
    )
    if resp.status_code != 200:
        print(f"    ERROR step 1 for {filename}: {_response_error_summary(resp)}")
        return None

    upload_info = resp.json()
    upload_url = upload_info["upload_url"]
    upload_params = upload_info.get("upload_params", {})

    # Step 2: POST file to upload URL
    with open(filepath, "rb") as f:
        resp2 = requests.post(
            upload_url,
            data=upload_params,
            files={"file": (filename, f, content_type)},
            allow_redirects=False,
            timeout=120,
        )

    # Step 3: Confirm upload (handle redirect or direct response)
    if resp2.status_code in (200, 201):
        return resp2.json()["id"]
    elif resp2.status_code in (301, 302, 303):
        confirm_url = resp2.headers.get("Location")
        if confirm_url:
            confirm_url = _validate_canvas_response_url(
                confirm_url,
                config["domain"],
                "upload confirmation redirect",
            )
            resp3 = requests.get(confirm_url, headers=config["headers"], timeout=30)
            if resp3.status_code == 200:
                return resp3.json()["id"]
            else:
                print(f"    ERROR step 3 for {filename}: {_response_error_summary(resp3)}")
                return None
    else:
        print(f"    ERROR step 2 for {filename}: {_response_error_summary(resp2)}")
        return None


# ============================================================
# IMAGE UPLOAD HELPER
# ============================================================
def upload_image(config, filepath, module_number=None, content_type=None):
    """Upload an image to Canvas Files with organized folder structure.

    Images are stored in: course files/images/m{N}/ (or images/shared/ if no module).

    Args:
        config: Canvas API config dict from get_config()
        filepath: Local path to the image file
        module_number: Module number (int) for folder organization, or None for shared/
        content_type: MIME type override. Auto-detected from extension if None.

    Returns:
        dict with 'file_id' and 'embed_url' on success, None on failure
    """
    require_course_id(config)

    # Auto-detect content type from extension
    if content_type is None:
        ext = os.path.splitext(filepath)[1].lower()
        content_types = {
            '.png': 'image/png',
            '.jpg': 'image/jpeg',
            '.jpeg': 'image/jpeg',
            '.gif': 'image/gif',
            '.svg': 'image/svg+xml',
            '.webp': 'image/webp',
        }
        content_type = content_types.get(ext, 'image/png')

    # Create folder: images/m{N}/ or images/shared/
    subfolder = f"m{module_number}" if module_number is not None else "shared"
    folder_id = get_or_create_folder(config, "images", subfolder)
    if not folder_id:
        # Fallback: try creating the parent 'images' folder first
        get_or_create_folder(config, "course files", "images")
        folder_id = get_or_create_folder(config, "images", subfolder)
    if not folder_id:
        print(f"  ERROR: Could not create folder images/{subfolder}")
        return None

    # Upload using existing 3-step process
    file_id = upload_file(config, filepath, folder_id, content_type)
    if not file_id:
        return None

    embed_url = f"/courses/{config['course_id']}/files/{file_id}/preview"
    print(f"  Uploaded image: {os.path.basename(filepath)} → file_id={file_id}")
    return {
        "file_id": file_id,
        "embed_url": embed_url,
    }


# ============================================================
# PAGE HELPERS
# ============================================================
def get_page(config, page_slug):
    """Fetch a Canvas wiki page by slug. Returns page dict or None."""
    require_course_id(config)
    try:
        resp = _request_with_retry(
            requests.get,
            f"{config['course_url']}/pages/{page_slug}",
            config["headers"],
        )
    except requests.RequestException as exc:
        print(f"  ERROR fetching page {page_slug}: {exc}")
        return None
    if resp.status_code == 200:
        return resp.json()
    print(f"  ERROR fetching page {page_slug}: {resp.status_code}")
    return None


def update_page(config, page_slug, body, from_staging=False):
    """Update a Canvas wiki page's HTML body. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        page_slug: URL slug of the page to update
        body: New HTML body content
        from_staging: Set True when called after a staged+previewed artifact
            exists (suppresses the staging-gate warning). The gated push path
            (push_to_canvas.py) always passes from_staging=True.
    """
    if not from_staging:
        logging.warning(
            "update_page called without a staged artifact (from_staging=False). "
            "Canvas page bodies should be updated only after staging + preview. "
            "Pass from_staging=True when calling from an authorised push path."
        )
    require_course_id(config)
    try:
        resp = _request_with_retry(
            requests.put,
            f"{config['course_url']}/pages/{page_slug}",
            {**config["headers"], "Content-Type": "application/json"},
            json={"wiki_page": {"body": body}},
        )
    except requests.RequestException as exc:
        print(f"  ERROR updating page {page_slug}: {exc}")
        return False
    if resp.status_code == 200:
        return True
    print(f"  ERROR updating page {page_slug}: {_response_error_summary(resp)}")
    return False


def get_all_pages(config, include_body: bool = False):
    """Fetch all wiki pages for a course. Returns list of page dicts.

    Args:
        config: Canvas API config dict from get_config()
        include_body: When True, each returned page dict already contains its
            full HTML ``body`` field (Canvas ``include[]=body`` index param).
            This collapses the common N+1 pattern of listing pages then
            GETting each body individually into a single paginated call.
            Verified live on canvas.asu.edu: the index body is byte-identical
            to the per-page GET body.  Default False preserves the original
            call surface for callers that only need page metadata.

    Returns:
        List of page dicts. When include_body=True each dict has a ``body``
        key (may be None/empty for blank pages — handle identically to the
        old per-page GET result).
    """
    require_course_id(config)
    params = {"per_page": 100}
    if include_body:
        params["include[]"] = "body"
    return paginated_get(
        f"{config['course_url']}/pages",
        config["headers"],
        params=params,
        canvas_domain=config["domain"],
    )


# Thin alias — ``list_pages`` is the older public name; ``get_all_pages`` is
# canonical.  Both names are kept so external callers and tests continue to
# work without modification.
list_pages = get_all_pages


def get_syllabus(config) -> dict:
    """Fetch course syllabus_body HTML + page URL.

    Returns {'html': str, 'page_url': str, 'updated_at': str|None}.
    """
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        config["course_url"],
        config["headers"],
        params={"include[]": "syllabus_body"},
    )
    if not resp or resp.status_code != 200:
        return {"html": "", "page_url": "", "updated_at": None}

    course = resp.json()
    html = course.get("syllabus_body") or ""
    page_url = (
        course.get("html_url")
        or f"https://{config['domain']}/courses/{config['course_id']}/assignments/syllabus"
    )
    return {
        "html": html,
        "page_url": page_url,
        "updated_at": course.get("updated_at"),
    }


def get_module_overview_page(config, module_id) -> dict:
    """Fetch the Module Overview page for a module.

    Heuristic: first page item in module OR page titled with 'Overview' / 'Module N'.
    Returns {'html': str, 'page_url': str, 'page_slug': str, 'module_id': int} or None.
    """
    require_course_id(config)
    items = get_module_items(config, module_id)
    page_items = [item for item in items if item.get("type") == "Page" and item.get("page_url")]
    if not page_items:
        return None

    module_number_match = None
    for item in page_items:
        text = f"{item.get('title', '')} {item.get('page_url', '')}"
        module_number_match = module_number_match or re.search(
            r"\b(?:module|m)\s*[-:]?\s*(\d+)\b",
            text,
            re.IGNORECASE,
        )
    module_number = module_number_match.group(1) if module_number_match else None

    def score(item):
        title = (item.get("title") or "").lower()
        slug = (item.get("page_url") or "").lower()
        combined = f"{title} {slug}"
        if "overview" in combined:
            return 0
        if module_number and re.search(rf"\bmodule\s*{re.escape(module_number)}\b", title):
            return 1
        return 2

    selected = sorted(enumerate(page_items), key=lambda pair: (score(pair[1]), pair[0]))[0][1]
    page_slug = selected.get("page_url")
    page = get_page(config, page_slug)
    if not page:
        return None

    return {
        "html": page.get("body") or "",
        "page_url": page.get("html_url") or selected.get("html_url") or "",
        "page_slug": page.get("url") or page_slug,
        "module_id": int(module_id),
    }


# ============================================================
# FOLDER HELPERS
# ============================================================
def get_or_create_folder(config, parent_path, folder_name):
    """Find or create a folder under parent_path. Returns folder ID or None."""
    require_course_id(config)
    full_path = f"{parent_path}/{folder_name}" if parent_path else folder_name

    # Check if folder exists
    resp = requests.get(
        f"{config['course_url']}/folders/by_path/{full_path}",
        headers=config["headers"],
        timeout=30,
    )
    if resp.status_code == 200:
        folders = resp.json()
        if folders:
            folder = folders[-1]
            print(f"  Found folder: {full_path} (id={folder['id']})")
            return folder["id"]

    # Create it
    resp = requests.post(
        f"{config['course_url']}/folders",
        headers=config["headers"],
        json={"name": folder_name, "parent_folder_path": parent_path},
        timeout=30,
    )
    if resp.status_code in (200, 201):
        folder = resp.json()
        print(f"  Created folder: {full_path} (id={folder['id']})")
        return folder["id"]
    else:
        print(f"  ERROR creating folder {full_path}: {_response_error_summary(resp)}")
        return None


# ============================================================
# MEDIA HELPERS
# ============================================================
def get_media_entry_id(config, file_id):
    """Get the media_entry_id for a Canvas file. Returns string or None."""
    require_course_id(config)
    resp = requests.get(
        f"{config['course_url']}/files/{file_id}",
        headers=config["headers"],
        timeout=30,
    )
    if resp.status_code == 200:
        return resp.json().get("media_entry_id")
    return None


def upload_caption(config, media_entry_id, vtt_content, locale="en", kind="subtitles"):
    """Upload a VTT caption track to a Canvas media object.

    Returns (status_code, response_text).
    """
    resp = requests.put(
        f"{config['base_url']}/media_objects/{media_entry_id}/media_tracks",
        headers=config["headers"],
        json=[{
            "locale": locale,
            "content": vtt_content,
            "kind": kind,
        }],
        timeout=30,
    )
    return resp.status_code, _sanitize_error_text(getattr(resp, "text", ""))


# ============================================================
# Staging auto-clear helper (shared by all create_* functions)
# ============================================================

def _clear_staged_on_success(config, object_type, staged_key):
    """Best-effort clear of a staged artifact after a successful Canvas create.

    Called only when from_staging=True AND staged_key is provided AND the
    Canvas API call succeeded. Never raises — failures are logged as warnings.
    """
    try:
        import staging_manager  # noqa: PLC0415
        course_id = (config or {}).get("course_id")
        if not course_id:
            logger.warning(
                "_clear_staged_on_success: no course_id in config; "
                "staged artifact %s/%s not cleared.", object_type, staged_key
            )
            return
        result = staging_manager.clear_staged(course_id, object_type, staged_key)
        if not result.get("ok"):
            logger.warning(
                "_clear_staged_on_success: clear_staged returned not-ok for "
                "%s/%s: %s", object_type, staged_key, result
            )
    except Exception as exc:  # noqa: BLE001
        logger.warning(
            "_clear_staged_on_success: failed to clear staged %s/%s: %s",
            object_type, staged_key, exc
        )


# ============================================================
# PAGE CREATE / DELETE / PUBLISH
# ============================================================
def create_page(config, title, body, published=False, editing_roles="teachers",
                from_staging=False, staged_key=None):
    """Create a new wiki page in the course.

    Args:
        config: Canvas API config dict from get_config()
        title: Page title
        body: HTML body content
        published: Publish immediately
        editing_roles: Who can edit — "teachers", "students", "members", "public"
        from_staging: Set True when called after a staged+previewed artifact exists
            (suppresses the staging-gate warning). The gated push path
            (push_to_canvas.py) always passes from_staging=True.
        staged_key: When from_staging=True and this is provided, the corresponding
            staged artifact at staging_manager.clear_staged(course_id, "pages",
            staged_key) is cleared on success. Ignored on failure or when
            from_staging=False. Backward-compatible: defaults to None (no clear).

    Returns:
        Page dict on success, None on failure
    """
    if not from_staging:
        logging.warning(
            "create_page called without a staged artifact (from_staging=False). "
            "Canvas objects should be created only after staging + preview. "
            "Pass from_staging=True when calling from an authorised push path."
        )
    require_course_id(config)
    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/pages",
        config["headers"],
        json={"wiki_page": {"title": title, "body": body,
                            "published": published, "editing_roles": editing_roles}},
    )
    if resp and resp.status_code in (200, 201):
        page = resp.json()
        print(f"  Created page: {title} (url={page.get('url', 'unknown')})")
        if from_staging and staged_key:
            _clear_staged_on_success(config, "pages", staged_key)
        return page
    print(f"  ERROR creating page {title}: {_response_error_summary(resp)}")
    return None


def delete_page(config, page_slug, confirm=False):
    """Delete a wiki page. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        page_slug: URL slug of the page to delete
        confirm: Must be True to proceed. Raises PermissionError otherwise.
            A timestamped backup is written before the delete.

    Returns:
        True on success, False on API failure.
    """
    if not confirm:
        raise PermissionError(
            "delete_page requires confirm=True. This is a destructive Canvas "
            "operation that permanently removes a live page. Fetch the content "
            "first, review it, then call delete_page(..., confirm=True)."
        )
    require_course_id(config)
    # Backup before delete
    page = get_page(config, page_slug)
    body = (page or {}).get("body", "") or ""
    bm = _backup_manager()
    result = bm.save_backup(
        config["course_id"], "pages", page_slug, body,
        diff_summary=f"Pre-delete backup for page {page_slug}",
    )
    if not result.get("ok"):
        raise RuntimeError(
            f"delete_page: backup failed for page '{page_slug}': "
            f"{result.get('error', 'unknown error')}. Delete aborted."
        )
    resp = _request_with_retry(
        requests.delete, f"{config['course_url']}/pages/{page_slug}",
        config["headers"],
    )
    return resp and resp.status_code == 200


def publish_page(config, page_slug):
    """Publish a wiki page. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put, f"{config['course_url']}/pages/{page_slug}",
        config["headers"],
        json={"wiki_page": {"published": True}},
    )
    return resp and resp.status_code == 200


# ============================================================
# MODULE HELPERS
# ============================================================
def create_module(config, name, position=None, unlock_at=None,
                  require_sequential_progress=False, publish=False):
    """Create a new module in the course.

    Returns Module dict on success, None on failure.
    """
    require_course_id(config)
    payload = {"module": {"name": name}}
    if position is not None:
        payload["module"]["position"] = position
    if unlock_at:
        payload["module"]["unlock_at"] = unlock_at
    if require_sequential_progress:
        payload["module"]["require_sequential_progress"] = True

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/modules",
        config["headers"], json=payload,
    )
    if resp and resp.status_code == 200:
        module = resp.json()
        if publish:
            publish_module(config, module["id"])
        print(f"  Created module: {name} (id={module['id']})")
        return module
    print(f"  ERROR creating module {name}: {_response_error_summary(resp)}")
    return None


def publish_module(config, module_id):
    """Publish a module. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put, f"{config['course_url']}/modules/{module_id}",
        config["headers"],
        json={"module": {"published": True}},
    )
    return resp and resp.status_code == 200


def delete_module(config, module_id):
    """Delete a module. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.delete, f"{config['course_url']}/modules/{module_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


def add_module_item(config, module_id, title, item_type, content_id=None,
                    position=None, indent=0, external_url=None, new_tab=False):
    """Add an item to a module.

    Args:
        item_type: Page, Assignment, Quiz, Discussion, File,
                   SubHeader, ExternalUrl, ExternalTool
        content_id: Canvas object ID, or page slug for Page type
        indent: Indentation level (0-3)

    Returns Module item dict on success, None on failure.
    """
    require_course_id(config)
    item = {"title": title, "type": item_type, "indent": indent}
    if content_id is not None:
        item["content_id"] = content_id
    if position is not None:
        item["position"] = position
    if external_url:
        item["external_url"] = external_url
    if new_tab:
        item["new_tab"] = True
    # For Page type, use page_url instead of content_id when a slug is passed
    if item_type == "Page" and isinstance(content_id, str) and not content_id.isdigit():
        del item["content_id"]
        item["page_url"] = content_id

    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/modules/{module_id}/items",
        config["headers"], json={"module_item": item},
    )
    if resp and resp.status_code == 200:
        print(f"    Added module item: {title} ({item_type})")
        return resp.json()
    print(f"    ERROR adding module item {title}: {_response_error_summary(resp)}")
    return None


# ============================================================
# QUIZ HELPERS
# ============================================================
VALID_SCORING_POLICIES = ("keep_highest", "keep_latest", "take_average")


def create_quiz(config, title, description="", quiz_type="assignment",
                time_limit=None, allowed_attempts=1, shuffle_answers=True,
                show_correct_answers=True, published=False, due_at=None,
                assignment_group_id=None, points_possible=None,
                scoring_policy=None, from_staging=False, staged_key=None):
    """Create a new quiz in the course.

    Args:
        quiz_type: "assignment" (graded), "practice_quiz", "graded_survey", "survey"
        time_limit: Minutes, or None for unlimited
        allowed_attempts: Number of attempts (-1 for unlimited)
        scoring_policy: One of "keep_highest", "keep_latest", "take_average".
            Only meaningful when allowed_attempts != 1. Passing this here
            closes the prior gap that forced a follow-up update_quiz() call.
        from_staging: Set True when called after a staged+previewed artifact exists
            (suppresses the staging-gate warning).
        staged_key: When from_staging=True and this is provided, the corresponding
            staged artifact at staging_manager.clear_staged(course_id, "quizzes",
            staged_key) is cleared on success. Backward-compatible: defaults to None.

    Returns Quiz dict on success, None on failure.
    """
    if not from_staging:
        logging.warning(
            "create_quiz called without a staged artifact (from_staging=False). "
            "Canvas objects should be created only after staging + preview. "
            "Pass from_staging=True when calling from an authorised push path."
        )
    require_course_id(config)
    quiz = {
        "title": title, "description": description, "quiz_type": quiz_type,
        "shuffle_answers": shuffle_answers,
        "show_correct_answers": show_correct_answers,
        "allowed_attempts": allowed_attempts, "published": published,
    }
    if time_limit is not None:
        quiz["time_limit"] = time_limit
    if due_at:
        quiz["due_at"] = due_at
    if assignment_group_id:
        quiz["assignment_group_id"] = assignment_group_id
    if points_possible is not None:
        quiz["points_possible"] = points_possible
    if scoring_policy is not None:
        if scoring_policy not in VALID_SCORING_POLICIES:
            raise ValueError(
                f"scoring_policy must be one of {VALID_SCORING_POLICIES}, "
                f"got {scoring_policy!r}"
            )
        quiz["scoring_policy"] = scoring_policy

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/quizzes",
        config["headers"], json={"quiz": quiz},
    )
    if resp and resp.status_code == 200:
        q = resp.json()
        print(f"  Created quiz: {title} (id={q['id']})")
        if from_staging and staged_key:
            _clear_staged_on_success(config, "quizzes", staged_key)
        return q
    print(f"  ERROR creating quiz {title}: {_response_error_summary(resp)}")
    return None


def _normalize_quiz_answers_for_write(answers):
    """Map Canvas read/staging answer feedback keys to Canvas write keys.

    Also maps numerical/range read-format keys to write-format keys:
      exact       -> answer_exact
      margin      -> answer_error_margin
      approximate -> answer_approximate
      precision   -> answer_precision
      start       -> answer_range_start
      end         -> answer_range_end
    Only maps when the read key is present and the write key is absent,
    so callers that already supply write-format values are not clobbered.
    """
    if not isinstance(answers, list):
        return answers

    # Read key -> write key for numerical/range answers
    _NUMERICAL_KEY_MAP = {
        "exact": "answer_exact",
        "margin": "answer_error_margin",
        "approximate": "answer_approximate",
        "precision": "answer_precision",
        "start": "answer_range_start",
        "end": "answer_range_end",
    }

    normalized = []
    for answer in answers:
        if not isinstance(answer, dict):
            normalized.append(answer)
            continue
        item = dict(answer)
        # Feedback key normalization
        feedback = (
            item.get("answer_comments")
            or item.get("answer_comment")
            or item.get("comments")
            or item.get("feedback")
        )
        if feedback:
            item["answer_comments"] = feedback
        feedback_html = (
            item.get("answer_comment_html")
            or item.get("comments_html")
            or item.get("answer_comments_html")
        )
        if feedback_html:
            item["answer_comment_html"] = feedback_html
        # Numerical/range key normalization: map read keys -> write keys
        for read_key, write_key in _NUMERICAL_KEY_MAP.items():
            if read_key in item and write_key not in item:
                item[write_key] = item[read_key]
        normalized.append(item)
    return normalized


def _cast_int_when_possible(value):
    """Return ints for integral Canvas floats, otherwise preserve value."""
    if isinstance(value, float) and value.is_integer():
        return int(value)
    if isinstance(value, str):
        try:
            parsed = float(value)
        except ValueError:
            return value
        if parsed.is_integer():
            return int(parsed)
    return value


def _strip_html_fragment(value):
    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", str(value or ""))).strip()


def _normalize_quiz_answers_for_read(answers, question_type=None, strip_html=False):
    """Map Canvas quiz answer read-shape to ASUCB staged answer shape.

    Canvas returns answer fields such as text/html, weight, comments, and
    comments_html. Staged JSON uses answer_text, answer_weight, and
    answer_comment_html/answer_comments_html. Question-type branches preserve
    matching, ordering, numerical, and fill-in-multiple-blanks fields.
    """
    if not isinstance(answers, list):
        return []
    normalized = []
    q_type = str(question_type or "")
    for answer in answers:
        if not isinstance(answer, dict):
            continue
        item = {}
        if answer.get("id") is not None:
            item["id"] = answer.get("id")

        raw_text = (
            answer.get("answer_text")
            or answer.get("text")
            or answer.get("html")
            or answer.get("answer_html")
            or ""
        )
        if strip_html:
            raw_text = _strip_html_fragment(raw_text)
        if raw_text != "":
            item["answer_text"] = raw_text

        if "answer_weight" in answer:
            item["answer_weight"] = _cast_int_when_possible(answer.get("answer_weight"))
        elif "weight" in answer:
            item["answer_weight"] = _cast_int_when_possible(answer.get("weight"))

        comments = (
            answer.get("answer_comments")
            or answer.get("answer_comment")
            or answer.get("comments")
            or answer.get("feedback")
        )
        if comments:
            item["answer_comments"] = comments
        comments_html = (
            answer.get("answer_comment_html")
            or answer.get("comments_html")
            or answer.get("answer_comments_html")
        )
        if comments_html:
            item["answer_comment_html"] = comments_html
            item["answer_comments_html"] = comments_html

        if q_type == "matching_question":
            left = (
                answer.get("answer_match_left")
                or answer.get("match_left")
                or answer.get("left")
                or answer.get("text")
                or item.get("answer_text")
            )
            right = (
                answer.get("answer_match_right")
                or answer.get("match_right")
                or answer.get("right")
                or answer.get("match_answer")
                or answer.get("matching_answer")
            )
            if strip_html:
                left = _strip_html_fragment(left)
                right = _strip_html_fragment(right)
            if left:
                item["answer_match_left"] = left
            if right:
                item["answer_match_right"] = right
        elif q_type == "fill_in_multiple_blanks_question":
            blank_id = answer.get("blank_id") or answer.get("blank") or answer.get("variable")
            if blank_id:
                item["blank_id"] = blank_id
        elif q_type == "numerical_question":
            for key in (
                "numerical_answer_type",
                "exact",
                "margin",
                "approximate",
                "precision",
                "start",
                "end",
            ):
                if key in answer and answer[key] is not None:
                    item[key] = _cast_int_when_possible(answer[key])
        elif q_type == "ordering_question":
            if "answer_order" in answer:
                item["answer_order"] = _cast_int_when_possible(answer["answer_order"])
            elif "position" in answer:
                item["answer_order"] = _cast_int_when_possible(answer["position"])

        normalized.append(item)
    return normalized


def add_quiz_question(config, quiz_id, question_name, question_text,
                      question_type="multiple_choice_question",
                      points_possible=1, answers=None, position=None,
                      **extra_fields):
    """Add a question to a quiz.

    For MC answers: [{"answer_text": "...", "answer_weight": 100}, ...]
    (weight 100 = correct, 0 = incorrect)

    Returns Question dict on success, None on failure.
    """
    require_course_id(config)
    q = {
        "question_name": question_name, "question_text": question_text,
        "question_type": question_type, "points_possible": points_possible,
    }
    for key in (
        "correct_comments",
        "incorrect_comments",
        "neutral_comments",
        "correct_comments_html",
        "incorrect_comments_html",
        "neutral_comments_html",
        "text_after_answers",
    ):
        if key in extra_fields and extra_fields[key] is not None:
            q[key] = extra_fields[key]
    if answers:
        q["answers"] = _normalize_quiz_answers_for_write(answers)
    if position is not None:
        q["position"] = position

    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/quizzes/{quiz_id}/questions",
        config["headers"], json={"question": q},
    )
    if resp and resp.status_code == 200:
        print(f"    Added question: {question_name}")
        return resp.json()
    print(f"    ERROR adding question {question_name}: {_response_error_summary(resp)}")
    return None


def get_question_bank(config, bank_id):
    """Fetch a question bank by ID. Returns bank dict, or None on 404/failure."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/question_banks/{bank_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    if resp and resp.status_code == 404:
        return None
    print(f"  ERROR fetching question bank {bank_id}: {_response_error_summary(resp)}")
    return None


def add_question_from_bank(config, quiz_id, question_bank_id, position=None):
    """Add a single question from a question bank to a quiz."""
    require_course_id(config)
    data = {"question[question_bank_id]": question_bank_id}
    if position is not None:
        data["question[position]"] = position

    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/quizzes/{quiz_id}/questions",
        config["headers"], data=data,
    )
    if resp and resp.status_code in (200, 201):
        return resp.json()
    print(f"  ERROR adding question from bank {question_bank_id}: {_response_error_summary(resp)}")
    return None


def create_quiz_group(config, quiz_id, name, pick_count, question_points,
                      assessment_question_bank_id):
    """Create a quiz group linked to an assessment question bank."""
    require_course_id(config)
    group = {
        "name": name,
        "pick_count": pick_count,
        "question_points": question_points,
        "assessment_question_bank_id": assessment_question_bank_id,
    }
    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/quizzes/{quiz_id}/groups",
        config["headers"], json={"quiz_groups": [group]},
    )
    if resp and resp.status_code in (200, 201):
        groups = resp.json().get("quiz_groups", [])
        return groups[0] if groups else None
    print(f"  ERROR creating quiz group {name}: {_response_error_summary(resp)}")
    return None


def update_quiz_question(config, quiz_id, question_id, **fields):
    """Update an existing quiz question. Returns question dict or None.

    WARNING: Canvas replaces the question on PUT. Callers MUST pass the full
    question (question_text, question_type, points_possible, answers) or
    unspecified fields will be silently wiped. Partial updates — e.g. only
    setting question_text — will reset answers to empty/zero.
    """
    require_course_id(config)
    if "question_text" in fields and "answers" not in fields:
        logger.warning(
            "update_quiz_question: 'question_text' provided without 'answers'. "
            "Canvas replaces the full question on PUT — existing answers will be "
            "wiped unless you include them explicitly. Pass the complete question "
            "payload to avoid silent data loss."
        )
    if "answers" in fields:
        fields = dict(fields)
        fields["answers"] = _normalize_quiz_answers_for_write(fields["answers"])
    resp = _request_with_retry(
        requests.put,
        f"{config['course_url']}/quizzes/{quiz_id}/questions/{question_id}",
        config["headers"],
        json={"question": fields},
    )
    if resp and resp.status_code == 200:
        return resp.json()
    print(f"  ERROR updating quiz question {question_id}: "
          f"{resp.status_code if resp else 'no response'}")
    return None


def delete_quiz_question(config, quiz_id, question_id):
    """Delete a quiz question. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.delete,
        f"{config['course_url']}/quizzes/{quiz_id}/questions/{question_id}",
        config["headers"],
    )
    if resp and resp.status_code in (200, 204):
        return True
    print(f"  ERROR deleting quiz question {question_id}: "
          f"{resp.status_code if resp else 'no response'}")
    return False


def delete_quiz(config, quiz_id, confirm=False):
    """Delete a quiz. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        quiz_id: Numeric quiz ID
        confirm: Must be True to proceed. Raises PermissionError otherwise.
            A timestamped backup is written before the delete.

    Returns:
        True on success, False on API failure.
    """
    if not confirm:
        raise PermissionError(
            "delete_quiz requires confirm=True. This is a destructive Canvas "
            "operation that permanently removes a live quiz. Fetch the content "
            "first, review it, then call delete_quiz(..., confirm=True)."
        )
    require_course_id(config)
    # Backup before delete
    quiz = get_quiz(config, quiz_id)
    bm = _backup_manager()
    result = bm.save_backup(
        config["course_id"], "quizzes", str(quiz_id), quiz or {},
        diff_summary=f"Pre-delete backup for quiz {quiz_id}",
    )
    if not result.get("ok"):
        raise RuntimeError(
            f"delete_quiz: backup failed for quiz '{quiz_id}': "
            f"{result.get('error', 'unknown error')}. Delete aborted."
        )
    resp = _request_with_retry(
        requests.delete, f"{config['course_url']}/quizzes/{quiz_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


def publish_quiz(config, quiz_id):
    """Publish a quiz. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put, f"{config['course_url']}/quizzes/{quiz_id}",
        config["headers"],
        json={"quiz": {"published": True}},
    )
    return resp and resp.status_code == 200


# ============================================================
# ASSIGNMENT HELPERS
# ============================================================
def create_assignment(config, name, description="", points_possible=100,
                      grading_type="points", submission_types=None,
                      due_at=None, lock_at=None, unlock_at=None,
                      published=False, assignment_group_id=None,
                      allowed_attempts=-1, peer_reviews=False,
                      turnitin_enabled=False, rubric_id=None,
                      from_staging=False, staged_key=None):
    """Create a new assignment in the course.

    Args:
        grading_type: points, percent, letter_grade, gpa_scale, pass_fail, not_graded
        submission_types: List — ["online_text_entry"], ["online_upload"], etc.
        rubric_id: Attach an existing rubric by ID after creation
        from_staging: Set True when called after a staged+previewed artifact exists
            (suppresses the staging-gate warning).
        staged_key: When from_staging=True and this is provided, the corresponding
            staged artifact at staging_manager.clear_staged(course_id, "assignments",
            staged_key) is cleared on success. Backward-compatible: defaults to None.

    Returns Assignment dict on success, None on failure.
    """
    if not from_staging:
        logging.warning(
            "create_assignment called without a staged artifact (from_staging=False). "
            "Canvas objects should be created only after staging + preview. "
            "Pass from_staging=True when calling from an authorised push path."
        )
    require_course_id(config)
    if submission_types is None:
        submission_types = ["online_text_entry"]

    asn = {
        "name": name, "description": description,
        "points_possible": points_possible, "grading_type": grading_type,
        "submission_types": submission_types, "published": published,
        "allowed_attempts": allowed_attempts,
    }
    if due_at:
        asn["due_at"] = due_at
    if lock_at:
        asn["lock_at"] = lock_at
    if unlock_at:
        asn["unlock_at"] = unlock_at
    if assignment_group_id:
        asn["assignment_group_id"] = assignment_group_id
    if peer_reviews:
        asn["peer_reviews"] = True
    if turnitin_enabled:
        asn["turnitin_enabled"] = True

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/assignments",
        config["headers"], json={"assignment": asn},
    )
    if resp and resp.status_code in (200, 201):
        assignment = resp.json()
        print(f"  Created assignment: {name} (id={assignment['id']})")
        if rubric_id:
            associate_rubric(config, assignment["id"], rubric_id)
        if from_staging and staged_key:
            _clear_staged_on_success(config, "assignments", staged_key)
        return assignment
    print(f"  ERROR creating assignment {name}: {_response_error_summary(resp)}")
    return None


def delete_assignment(config, assignment_id, confirm=False):
    """Delete an assignment. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        assignment_id: Numeric assignment ID
        confirm: Must be True to proceed. Raises PermissionError otherwise.
            A timestamped backup is written before the delete.

    Returns:
        True on success, False on API failure.
    """
    if not confirm:
        raise PermissionError(
            "delete_assignment requires confirm=True. This is a destructive Canvas "
            "operation that permanently removes a live assignment. Fetch the content "
            "first, review it, then call delete_assignment(..., confirm=True)."
        )
    require_course_id(config)
    # Backup before delete
    assignment = get_assignment(config, assignment_id)
    content = (assignment or {}).get("description", "") or ""
    bm = _backup_manager()
    result = bm.save_backup(
        config["course_id"], "assignments", str(assignment_id), content,
        diff_summary=f"Pre-delete backup for assignment {assignment_id}",
    )
    if not result.get("ok"):
        raise RuntimeError(
            f"delete_assignment: backup failed for assignment '{assignment_id}': "
            f"{result.get('error', 'unknown error')}. Delete aborted."
        )
    resp = _request_with_retry(
        requests.delete, f"{config['course_url']}/assignments/{assignment_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


def publish_assignment(config, assignment_id):
    """Publish an assignment. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put, f"{config['course_url']}/assignments/{assignment_id}",
        config["headers"],
        json={"assignment": {"published": True}},
    )
    return resp and resp.status_code == 200


# ============================================================
# DISCUSSION HELPERS
# ============================================================
def create_discussion(config, title, message="", discussion_type="threaded",
                      require_initial_post=False, published=False,
                      pinned=False, locked=False, due_at=None,
                      assignment=None, from_staging=False, staged_key=None):
    """Create a new discussion topic in the course.

    Args:
        discussion_type: "side_comment", "threaded", or "not_threaded"
        published: Whether the discussion is published immediately.
            Defaults to False.
        assignment: Dict for graded discussion:
            {"points_possible": N, "grading_type": "points"}
        from_staging: Set True when called after a staged+previewed artifact exists
            (suppresses the staging-gate warning).
        staged_key: When from_staging=True and this is provided, the corresponding
            staged artifact at staging_manager.clear_staged(course_id, "discussions",
            staged_key) is cleared on success. Backward-compatible: defaults to None.

    Returns Discussion dict on success, None on failure.
    """
    if not from_staging:
        logging.warning(
            "create_discussion called without a staged artifact (from_staging=False). "
            "Canvas objects should be created only after staging + preview. "
            "Pass from_staging=True when calling from an authorised push path."
        )
    require_course_id(config)
    payload = {
        "title": title, "message": message,
        "discussion_type": discussion_type,
        "require_initial_post": require_initial_post,
        "published": published, "pinned": pinned, "locked": locked,
    }
    if due_at:
        payload["due_at"] = due_at
    if assignment:
        payload["assignment"] = assignment

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/discussion_topics",
        config["headers"], json=payload,
    )
    if resp and resp.status_code in (200, 201):
        disc = resp.json()
        print(f"  Created discussion: {title} (id={disc['id']})")
        if from_staging and staged_key:
            _clear_staged_on_success(config, "discussions", staged_key)
        return disc
    print(f"  ERROR creating discussion {title}: {_response_error_summary(resp)}")
    return None


def delete_discussion(config, discussion_id, confirm=False):
    """Delete a discussion topic. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        discussion_id: Numeric discussion topic ID
        confirm: Must be True to proceed. Raises PermissionError otherwise.
            A timestamped backup is written before the delete.

    Returns:
        True on success, False on API failure.
    """
    if not confirm:
        raise PermissionError(
            "delete_discussion requires confirm=True. This is a destructive Canvas "
            "operation that permanently removes a live discussion topic. Fetch the "
            "content first, review it, then call delete_discussion(..., confirm=True)."
        )
    require_course_id(config)
    # Backup before delete
    disc = get_discussion(config, discussion_id)
    content = (disc or {}).get("message", "") or ""
    bm = _backup_manager()
    result = bm.save_backup(
        config["course_id"], "discussions", str(discussion_id), content,
        diff_summary=f"Pre-delete backup for discussion {discussion_id}",
    )
    if not result.get("ok"):
        raise RuntimeError(
            f"delete_discussion: backup failed for discussion '{discussion_id}': "
            f"{result.get('error', 'unknown error')}. Delete aborted."
        )
    resp = _request_with_retry(
        requests.delete,
        f"{config['course_url']}/discussion_topics/{discussion_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


# ============================================================
# RUBRIC HELPERS
# ============================================================
def create_rubric(config, title, criteria, points_possible=None,
                  assignment_id=None, free_form_comments=False, bookmark=False):
    """Create a rubric and optionally associate it with an assignment or course.

    Note: API-created rubrics do NOT appear on the course Rubrics page until
    they have a rubric_association. Pass ``bookmark=True`` (when no
    ``assignment_id`` is given) to post a Course-context bookmark association
    automatically, making the rubric visible on the Rubrics index. When an
    ``assignment_id`` is supplied the grading association already satisfies
    visibility; ``bookmark`` is ignored in that case.

    IMPORTANT -- rating long_description is DROPPED by Canvas on write:
    Sending ``long_description`` in a rating returns HTTP 200 but stores "".
    Always bake the full cell text into the rating ``description`` field.
    Do NOT pass ``long_description`` on ratings -- it will be silently lost.

    Args:
        criteria: List of criterion dicts:
            [{"description": "Name", "points": 25,
              "ratings": [{"description": "Exemplary", "points": 25}, ...]}]
        assignment_id: Associate with this assignment ID (grading association).
        bookmark: When True and no assignment_id is given, post a Course
            bookmark association so the rubric is visible on the Rubrics page.

    Returns Rubric dict on success, None on failure.
    """
    require_course_id(config)
    if points_possible is None:
        points_possible = sum(c.get("points", 0) for c in criteria)

    # Build criteria in Canvas's indexed format: {"0": {...}, "1": {...}}
    criteria_payload = {}
    for i, criterion in enumerate(criteria):
        c = {"description": criterion["description"],
             "points": criterion.get("points", 0)}
        if "long_description" in criterion:
            c["long_description"] = criterion["long_description"]
        if "ratings" in criterion:
            c["ratings"] = {
                str(j): {"description": r["description"], "points": r["points"]}
                for j, r in enumerate(criterion["ratings"])
            }
        criteria_payload[str(i)] = c

    payload = {
        "rubric": {
            "title": title, "points_possible": points_possible,
            "free_form_criterion_comments": free_form_comments,
            "criteria": criteria_payload,
        }
    }
    if assignment_id:
        payload["rubric_association"] = {
            "association_id": assignment_id,
            "association_type": "Assignment",
            "use_for_grading": True, "purpose": "grading",
        }

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/rubrics",
        config["headers"], json=payload,
    )
    if resp and resp.status_code in (200, 201):
        rubric = resp.json()
        rubric_data = rubric.get("rubric", rubric)
        rubric_id = rubric_data.get("id")
        print(f"  Created rubric: {title} (id={rubric_id or 'unknown'})")
        # Post a Course-context bookmark so the rubric appears on the Rubrics
        # page when no assignment association (which provides visibility) was
        # requested alongside creation.
        if bookmark and not assignment_id and rubric_id:
            assoc_resp = _request_with_retry(
                requests.post,
                f"{config['course_url']}/rubric_associations",
                config["headers"],
                json={"rubric_association": {
                    "rubric_id": rubric_id,
                    "association_id": config["course_id"],
                    "association_type": "Course",
                    "purpose": "bookmark",
                    "use_for_grading": False,
                }},
            )
            if assoc_resp and assoc_resp.status_code in (200, 201):
                print(f"    Bookmarked rubric {rubric_id} on course Rubrics page")
            else:
                print(f"    WARNING: rubric created but Course bookmark failed: "
                      f"{_response_error_summary(assoc_resp)}")
        return rubric
    print(f"  ERROR creating rubric {title}: {_response_error_summary(resp)}")
    return None


def associate_rubric(config, assignment_id, rubric_id, use_for_grading=True):
    """Associate an existing rubric with an assignment."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/rubric_associations",
        config["headers"],
        json={"rubric_association": {
            "rubric_id": rubric_id, "association_id": assignment_id,
            "association_type": "Assignment",
            "use_for_grading": use_for_grading, "purpose": "grading",
        }},
    )
    if resp and resp.status_code in (200, 201):
        print(f"    Associated rubric {rubric_id} with assignment {assignment_id}")
        return resp.json()
    print(f"    ERROR associating rubric: "
          f"{resp.status_code if resp else 'no response'}")
    return None


def update_rubric(config, rubric_id, title, criteria, points_possible=None,
                  free_form_comments=False):
    """Replace an existing rubric via delete + recreate + re-associate.

    Canvas does not persist criteria edits on already-associated rubrics (PUT
    returns 200 but a fresh GET shows the original). The only reliable update
    path is: read existing associations, delete the old rubric, create the new
    one, re-attach every prior association preserving use_for_grading.

    The Course-bookmark 500 that sometimes follows the create step is transient
    and non-fatal -- the rubric is visible through assignment associations.

    Args:
        rubric_id: ID of the rubric to replace.
        title: New title for the replacement rubric.
        criteria: New criteria list (same format as create_rubric).
        points_possible: Total points; inferred from criteria if None.
        free_form_comments: Passed to create_rubric.

    Returns:
        dict with keys "rubric" (new rubric dict) and "associations"
        (list of re-association response dicts), or None on failure.
    """
    require_course_id(config)

    # 1. Read existing rubric_associations so we can re-attach them.
    #    Rubric reads are course-scoped; base_url (/api/v1) would 404 and
    #    silently drop every association before the delete below.
    assoc_resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/rubrics/{rubric_id}",
        config["headers"],
        params={"include[]": "associations"},
    )
    existing_associations = []
    if assoc_resp and assoc_resp.status_code == 200:
        data = assoc_resp.json()
        existing_associations = data.get("associations", [])
    else:
        print(f"  WARNING: could not read rubric {rubric_id} associations "
              f"before delete: {_response_error_summary(assoc_resp)}")

    # 2. Delete the old rubric.
    del_resp = _request_with_retry(
        requests.delete,
        f"{config['course_url']}/rubrics/{rubric_id}",
        config["headers"],
    )
    if del_resp and del_resp.status_code in (200, 201, 204):
        print(f"  Deleted rubric {rubric_id} for replacement")
    else:
        print(f"  ERROR deleting rubric {rubric_id}: "
              f"{_response_error_summary(del_resp)}")
        return None

    # 3. Create the new rubric (no assignment_id -- we re-associate below).
    new_rubric = create_rubric(
        config, title, criteria,
        points_possible=points_possible,
        free_form_comments=free_form_comments,
        bookmark=False,
    )
    if not new_rubric:
        return None

    rubric_data = new_rubric.get("rubric", new_rubric)
    new_id = rubric_data.get("id")

    # 4. Re-attach every prior association preserving use_for_grading.
    new_associations = []
    for assoc in existing_associations:
        assoc_type = assoc.get("association_type", "")
        if assoc_type != "Assignment":
            # Skip Course-bookmark associations; a new one will be auto-created
            # if bookmark=True is desired -- caller can re-request it.
            continue
        assignment_id = assoc.get("association_id")
        use_for_grading = assoc.get("use_for_grading", True)
        if assignment_id and new_id:
            result = associate_rubric(
                config, assignment_id, new_id,
                use_for_grading=use_for_grading,
            )
            if result:
                new_associations.append(result)

    return {"rubric": rubric_data, "associations": new_associations}


# ============================================================
# ASSIGNMENT GROUP HELPERS
# ============================================================
def create_assignment_group(config, name, position=None, group_weight=None):
    """Create an assignment group (grading category).

    Returns Assignment group dict on success, None on failure.
    """
    require_course_id(config)
    payload = {"name": name}
    if position is not None:
        payload["position"] = position
    if group_weight is not None:
        payload["group_weight"] = group_weight

    resp = _request_with_retry(
        requests.post, f"{config['course_url']}/assignment_groups",
        config["headers"], json=payload,
    )
    if resp and resp.status_code in (200, 201):
        group = resp.json()
        print(f"  Created assignment group: {name} (id={group['id']})")
        return group
    print(f"  ERROR creating assignment group {name}: "
          f"{resp.status_code if resp else 'no response'}")
    return None


def get_assignment_groups(config, include_assignments=False):
    """Get all assignment groups for a course."""
    require_course_id(config)
    url = f"{config['course_url']}/assignment_groups?per_page=100"
    if include_assignments:
        url += "&include[]=assignments"
    return paginated_get(url, config["headers"], canvas_domain=config["domain"])


# ============================================================
# MODULE RETRIEVAL HELPERS
# ============================================================
def get_modules(config, include_items=False):
    """Get all modules for a course, optionally with items.

    Returns list of module dicts.
    """
    require_course_id(config)
    url = f"{config['course_url']}/modules?per_page=100"
    if include_items:
        url += "&include[]=items"
    return paginated_get(url, config["headers"], canvas_domain=config["domain"])


def list_modules(config):
    """List all modules for a course, including module items.

    Thin alias — equivalent to ``get_modules(config, include_items=True)``.
    Both names are kept so external callers and tests continue to work.
    """
    return get_modules(config, include_items=True)


def get_module_items(config, module_id):
    """Get all items in a specific module."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/modules/{module_id}/items?per_page=100",
        config["headers"],
        canvas_domain=config["domain"],
    )


def delete_module_item(config, module_id, item_id):
    """Delete a module item. Returns True on success."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.delete,
        f"{config['course_url']}/modules/{module_id}/items/{item_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


# ============================================================
# ASSIGNMENT / QUIZ / DISCUSSION RETRIEVAL HELPERS
# ============================================================
def list_assignments(config):
    """List all assignments for a course."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/assignments?per_page=100",
        config["headers"],
        canvas_domain=config["domain"],
    )


def list_quizzes(config):
    """List all quizzes for a course."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/quizzes?per_page=100",
        config["headers"],
        canvas_domain=config["domain"],
    )


def list_discussions(config):
    """List all discussion topics for a course."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/discussion_topics?per_page=100",
        config["headers"],
        canvas_domain=config["domain"],
    )


def list_announcements(config):
    """List all announcements for a course (discussion_topics with only_announcements=true)."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/discussion_topics",
        config["headers"],
        params={"per_page": 100, "only_announcements": "true"},
        canvas_domain=config["domain"],
    )


def get_folder(config, folder_id):
    """Fetch a single Canvas folder's metadata by ID. Returns dict (with full_name) or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.get, f"{config['base_url']}/folders/{folder_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


def list_files(config, content_types=None):
    """List all Files for a course, optionally filtered by content_types (MIME prefixes)."""
    require_course_id(config)
    params = {"per_page": 100}
    if content_types:
        params["content_types[]"] = list(content_types)
    return paginated_get(
        f"{config['course_url']}/files",
        config["headers"],
        params=params,
        canvas_domain=config["domain"],
    )


def get_file(config, file_id):
    """Fetch a single Canvas file's metadata by ID. Returns dict or None.

    Uses the top-level Files#api_show endpoint (/api/v1/files/:id) rather
    than the course-scoped list — files have globally unique IDs, and this
    is the endpoint pattern already exercised live in
    tests/integration/test_live_file_upload.py.
    """
    require_course_id(config)
    resp = _request_with_retry(
        requests.get, f"{config['base_url']}/files/{file_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


def delete_file(config, file_id, confirm=False):
    """Delete a Canvas file. Returns True on success.

    Args:
        config: Canvas API config dict from get_config()
        file_id: Numeric Canvas file ID
        confirm: Must be True to proceed. Raises PermissionError otherwise.

    Unlike delete_page/delete_quiz/delete_assignment/delete_discussion, this
    does NOT auto-backup: file bytes are binary and backup_manager only
    stores text/JSON content. Callers that need a pre-delete backup of file
    bytes back it up themselves (see push_remediated_file.py, which backs up
    the live file before ever replacing it).

    Returns:
        True on success, False on API failure.
    """
    if not confirm:
        raise PermissionError(
            "delete_file requires confirm=True. This is a destructive Canvas "
            "operation that permanently removes a live file."
        )
    require_course_id(config)
    resp = _request_with_retry(
        requests.delete, f"{config['base_url']}/files/{file_id}",
        config["headers"],
    )
    return resp and resp.status_code == 200


def get_assignment(config, assignment_id):
    """Fetch a single assignment by ID. Returns assignment dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/assignments/{assignment_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


def get_quiz(config, quiz_id):
    """Fetch a single quiz by ID. Returns quiz dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/quizzes/{quiz_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


def get_quiz_questions(config, quiz_id):
    """Fetch all questions for a quiz."""
    require_course_id(config)
    return paginated_get(
        f"{config['course_url']}/quizzes/{quiz_id}/questions?per_page=100",
        config["headers"],
        canvas_domain=config["domain"],
    )


def get_discussion(config, topic_id):
    """Fetch a single discussion topic by ID. Returns topic dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/discussion_topics/{topic_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


def update_assignment(config, assignment_id, **fields):
    """Update assignment fields. Returns assignment dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put,
        f"{config['course_url']}/assignments/{assignment_id}",
        config["headers"],
        json={"assignment": fields},
    )
    if resp and resp.status_code == 200:
        return resp.json()
    print(f"  ERROR updating assignment {assignment_id}: "
          f"{resp.status_code if resp else 'no response'}")
    return None


def update_quiz(config, quiz_id, **fields):
    """Update quiz fields. Returns quiz dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put,
        f"{config['course_url']}/quizzes/{quiz_id}",
        config["headers"],
        json={"quiz": fields},
    )
    if resp and resp.status_code == 200:
        return resp.json()
    print(f"  ERROR updating quiz {quiz_id}: "
          f"{resp.status_code if resp else 'no response'}")
    return None


def update_discussion(config, topic_id, **fields):
    """Update discussion topic fields. Returns topic dict or None."""
    require_course_id(config)
    resp = _request_with_retry(
        requests.put,
        f"{config['course_url']}/discussion_topics/{topic_id}",
        config["headers"],
        json=fields,
    )
    if resp and resp.status_code == 200:
        return resp.json()
    print(f"  ERROR updating discussion {topic_id}: "
          f"{resp.status_code if resp else 'no response'}")
    return None


# ============================================================
# CONTENT MIGRATION (COURSE COPY)
# ============================================================
def create_content_migration(config, source_course_id, select=None):
    """Start a content migration (course copy) into the configured course.

    Args:
        config: Destination course config dict
        source_course_id: Source course ID to copy from
        select: Optional dict for selective copy, e.g.:
            {"modules": {"1234": True}, "pages": {"5678": True}}
            If None, copies everything.

    Returns migration dict on success, None on failure.
    """
    require_course_id(config)
    payload = {
        "migration_type": "course_copy_importer",
        "settings": {"source_course_id": str(source_course_id)},
    }
    if select:
        payload["select"] = select

    resp = _request_with_retry(
        requests.post,
        f"{config['course_url']}/content_migrations",
        config["headers"],
        json=payload,
    )
    if resp and resp.status_code in (200, 201):
        migration = resp.json()
        print(f"  Started content migration: id={migration['id']}, "
              f"state={migration.get('workflow_state', 'unknown')}")
        return migration
    print(f"  ERROR starting content migration: {_response_error_summary(resp)}")
    return None


def check_migration_progress(config, migration_id):
    """Check the status of a content migration.

    Returns dict with workflow_state (queued, running, completed, failed),
    completion percentage, and any issues.
    """
    require_course_id(config)
    resp = _request_with_retry(
        requests.get,
        f"{config['course_url']}/content_migrations/{migration_id}",
        config["headers"],
    )
    if resp and resp.status_code == 200:
        return resp.json()
    return None


# ============================================================
# FRIENDLY ERROR MESSAGES
# ============================================================
FRIENDLY_ERRORS = {
    401: "Authentication failed — check your Canvas token.",
    403: "Permission denied — your token may not have access to this course.",
    404: "Not found — the course, page, or item doesn't exist.",
    409: "Conflict — another edit may be in progress.",
    422: "Invalid data — Canvas rejected the request. Check field values.",
    429: "Rate limited — too many requests. Retrying automatically.",
    500: "Canvas server error — try again in a moment.",
    502: "Canvas gateway error — try again in a moment.",
    503: "Canvas is temporarily unavailable — try again shortly.",
    504: "Canvas timed out — the request may still be processing.",
}


def friendly_error(response):
    """Return a user-friendly error message for a Canvas API response."""
    if response is None:
        return "No response from Canvas — check your network connection."
    code = response.status_code
    default = f"Canvas returned HTTP {code}."
    return FRIENDLY_ERRORS.get(code, default)


# ============================================================
# PATH HELPERS (per-course storage)
# ============================================================
def get_plugin_root():
    """Return the ASUCB plugin root directory."""
    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def create_locked_module(config, name, items, prerequisite_module_id=None,
                         position=None, require_sequential_progress=False,
                         publish=True):
    """Create a Canvas module, add items, optionally lock behind another module.

    Args:
        config: Standard Canvas config dict.
        name: Module name.
        items: Ordered list of dicts with keys:
            title (str), type (str), ref (page slug | numeric id | None),
            completion ("must_view"|"must_submit"|"must_contribute"|
                        "must_mark_done"|None), indent (int, default 0),
            external_url (str, optional for ExternalUrl type).
        prerequisite_module_id: If set, lock this module behind it.
        position: Module position (1-based).
        require_sequential_progress: Enforce sequential item completion.
        publish: Publish the module after building it.

    Returns:
        dict with keys "module", "item_ids", "failed_items" on success,
        or None if the module could not be created.
    """
    mod = create_module(config, name, position=position,
                        require_sequential_progress=require_sequential_progress,
                        publish=False)
    if mod is None:
        print(f"  ERROR in create_locked_module: could not create module '{name}'")
        return None

    mod_id = mod["id"]
    item_ids = []
    failed_items = []
    completion_map = []  # list of (item_id, completion_type)

    for i, item in enumerate(items, start=1):
        result = add_module_item(
            config, mod_id,
            item["title"], item["type"],
            content_id=item.get("ref"),
            position=i,
            indent=item.get("indent", 0),
            external_url=item.get("external_url"),
        )
        if result is None:
            failed_items.append(item["title"])
        else:
            item_id = result["id"]
            item_ids.append(item_id)
            completion = item.get("completion")
            if completion is not None:
                completion_map.append((item_id, completion))

    if prerequisite_module_id is not None:
        _request_with_retry(
            requests.put,
            f"{config['course_url']}/modules/{mod_id}",
            config["headers"],
            json={"module": {
                "prerequisite_module_ids": [prerequisite_module_id],
                "require_sequential_progress": require_sequential_progress,
            }},
        )

    for item_id, completion in completion_map:
        _request_with_retry(
            requests.put,
            f"{config['course_url']}/modules/{mod_id}/items/{item_id}",
            config["headers"],
            json={"module_item": {"completion_requirement": {"type": completion}}},
        )

    if publish:
        publish_module(config, mod_id)

    return {"module": mod, "item_ids": item_ids, "failed_items": failed_items}


def get_course_dir(course_id):
    """Return the per-course state directory, creating it if needed."""
    root = get_plugin_root()
    course_dir = os.path.join(root, "courses", str(course_id))
    os.makedirs(course_dir, exist_ok=True)
    return course_dir
