team-tryouts/app/storage.py

"""Where the application's files live, and how the database refers to them.

Three roots: the project itself, the document store, and the log directory.
They are here together because they are the same defect three times over
(OBS-006) — `os.path.join(os.getcwd(), …)`, evaluated at import or upload
time, so every one of them moved with whatever directory the process was
started from. The document store was fixed first, in wave G, because it was
the one that also blocked OPS-011; the other two were left behind, which is
the repeated lesson of this project: a faulty pattern corrected in one layer
stays in the others.

Contracts were stored at `os.path.join(os.getcwd(), 'documents', …)`,
evaluated at upload time, and the resulting absolute path was written into
`Contract.file_path`. The storage root therefore moved with whatever
directory the process happened to be started from. Two consequences:

  - one latent: start the server from elsewhere and new contracts land in a
    new tree while the old ones become unreadable — with the database still
    saying they are there, so the failure surfaces as a 500 on download
    rather than as anything a person could act on;
  - one blocking: it rules out a release-directory deployment (OPS-011)
    outright. Every stored path would point inside a release that is about
    to be replaced, so the first switch would take every contract ever
    uploaded with it.

New rows keep a path *relative* to the document root. Old rows keep their
absolute path and are returned untouched, so this change needs no data
migration and can ship before Alembic does (DB-002).
"""

import io
import logging
import os

from app.google_drive import (
    GoogleDriveStorageError,
    delete_file as delete_google_drive_file,
    download_file as download_google_drive_file,
    upload_file as upload_google_drive_file,
)

#: Environment override for the document root. What a release-directory
#: deployment sets, to a path outside the releases — alongside them, not
#: inside whichever one is current.
DOCUMENTS_ROOT_ENV = 'DOCUMENTS_ROOT'

#: Environment override for the log directory. Same reasoning: a release
#: directory that carries its own logs loses them at the next switch.
LOG_DIR_ENV = 'LOG_DIR'

#: Environment override for the backup directory.
BACKUP_DIR_ENV = 'BACKUP_DIR'

#: Sub-directory holding uploaded contracts, under the document root.
CONTRACTS_DIR = 'contrats signés'

#: Database marker for documents stored remotely. Existing rows continue to
#: hold relative or absolute filesystem paths, so switching storage does not
#: invalidate contracts already uploaded before the Google Drive move.
GOOGLE_DRIVE_PATH_PREFIX = 'gdrive://'

#: Storage backends deliberately stay explicit. Google Drive is the
#: production default; local disk exists only for legacy rows and isolated
#: test/development environments.
DOCUMENT_STORAGE_BACKEND_ENV = 'DOCUMENT_STORAGE_BACKEND'


def project_root():
    """Absolute path of the project, derived from this file's location.

    The anchor every other root falls back to. Not `os.getcwd()`: the
    process is started by Waitress under Pterodactyl, by pytest, by a task
    scheduler and by a person in a shell, and only one of those four is
    reliably in the project directory.
    """
    package_dir = os.path.dirname(os.path.abspath(__file__))
    return os.path.dirname(package_dir)


def documents_root():
    """Absolute path of the document store.

    Falls back to `documents/` beside this package — the project root
    wherever it is installed, rather than wherever the process was launched.
    """
    return _rooted(DOCUMENTS_ROOT_ENV, 'documents')


def logs_root():
    """Absolute path of the log directory.

    Was `os.path.join(os.getcwd(), 'logs')`. Starting the server from
    another directory sent the logs somewhere new without a word, which is
    the worst possible failure mode for the one file you go and read when
    something else has gone wrong.
    """
    return _rooted(LOG_DIR_ENV, 'logs')


def backups_root():
    """Absolute path of the backup directory."""
    return _rooted(BACKUP_DIR_ENV, 'backups')


def _rooted(env_name, default_name):
    """The configured path, or `default_name` under the project root."""
    configured = os.getenv(env_name)
    if configured:
        return os.path.abspath(configured)
    return os.path.join(project_root(), default_name)


def document_storage_backend():
    """Return the configured backend for new document uploads."""
    backend = os.getenv(DOCUMENT_STORAGE_BACKEND_ENV, 'google_drive').strip().lower()
    if backend not in {'google_drive', 'local'}:
        raise ValueError(
            f'{DOCUMENT_STORAGE_BACKEND_ENV} must be "google_drive" or "local", not {backend!r}'
        )
    return backend


def is_google_drive_path(stored_path):
    """Whether a database path references a Google Drive file."""
    return bool(stored_path and stored_path.startswith(GOOGLE_DRIVE_PATH_PREFIX))


def _google_drive_id(stored_path):
    file_id = stored_path.removeprefix(GOOGLE_DRIVE_PATH_PREFIX)
    if not file_id:
        raise GoogleDriveStorageError('The stored Google Drive file identifier is empty.')
    return file_id


def store_uploaded_document(file_storage, relative_path):
    """Store an uploaded document and return its durable database reference.

    Local storage retains the relative-path format used by existing rows. A
    Google Drive upload returns an opaque Drive file identifier prefixed with
    ``gdrive://`` so it cannot be mistaken for a filesystem path.
    """
    if document_storage_backend() == 'local':
        absolute_path = document_path(relative_path)
        os.makedirs(os.path.dirname(absolute_path), exist_ok=True)
        file_storage.save(absolute_path)
        return relative_path

    file_id = upload_google_drive_file(
        stream=file_storage.stream,
        filename=os.path.basename(relative_path),
        mimetype=file_storage.mimetype or 'application/pdf',
    )
    return f'{GOOGLE_DRIVE_PATH_PREFIX}{file_id}'


def open_document(stored_path):
    """Return a filesystem path or in-memory stream suitable for ``send_file``."""
    if is_google_drive_path(stored_path):
        return io.BytesIO(download_google_drive_file(_google_drive_id(stored_path)))
    return document_path(stored_path)


def discard_documents(stored_paths):
    """Remove these documents from disk. Returns how many went (DATA-012).

    `delete_user` removed the Contract rows and left the PDFs. Signed,
    named contracts therefore stayed on the server after the account was
    deleted, with nothing in the database pointing at them — invisible to
    the application, unmanageable through it, and still personal data.

    Call this **after** the commit that removed the rows, never before: a
    failure between the two should leave a file with no row (recoverable,
    and what the previous behaviour produced anyway) rather than a row with
    no file (a download that 500s for ever).

    A path that cannot be removed is logged and skipped. Nothing here should
    be able to abort the deletion of an account.
    """
    logger = logging.getLogger(__name__)
    removed = 0
    for stored_path in stored_paths:
        if not stored_path:
            continue
        try:
            if is_google_drive_path(stored_path):
                delete_google_drive_file(_google_drive_id(stored_path))
            else:
                os.remove(document_path(stored_path))
            removed += 1
        except FileNotFoundError:
            # Already gone. Two contracts sharing a stem, or a previous
            # attempt: not a problem, and not worth an error line.
            logger.info('Document already absent: %s', stored_path)
        except (GoogleDriveStorageError, OSError) as exc:
            logger.error(
                'Could not remove %s (%s). It is now an orphan: no database row '
                'refers to it, so nothing in the application will ever offer to '
                'delete it again.',
                stored_path,
                exc,
            )
    return removed


def document_path(stored_path):
    """Absolute path of a document, from what the database holds.

    Args:
        stored_path: The value of Contract.file_path or signed_file_path.
            Relative for rows written since this module existed, absolute
            for the ones written before.

    Returns:
        str: An absolute path.
    """
    if os.path.isabs(stored_path):
        return stored_path
    return os.path.join(documents_root(), stored_path)