utility.py

"""

    Greynir: Natural language processing for Icelandic

    Copyright (C) 2023 Miðeind ehf.

       This program is free software: you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published by
       the Free Software Foundation, either version 3 of the License, or
       (at your option) any later version.
       This program is distributed in the hope that it will be useful,
       but WITHOUT ANY WARRANTY; without even the implied warranty of
       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
       GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see http://www.gnu.org/licenses/.


    Utility functions used in various places in the codebase.

"""
from typing import Any, Dict, List, Optional

import os
import json
import string
import logging
from functools import lru_cache
from pathlib import Path

from tts import DEFAULT_LOCALE, LOCALE_TO_VOICE_ID

# Path which points to the root folder of Greynir
GREYNIR_ROOT_DIR: Path = Path(__file__).parent.resolve()

# Other useful paths
CONFIG_DIR = GREYNIR_ROOT_DIR / "config"

RESOURCES_DIR = GREYNIR_ROOT_DIR / "resources"

# Tell icespeak where our API keys live. Its own default is a relative "keys"
# directory, which this project has never had -- the keys are in resources/.
# icespeak reads this at import time and asserts that at least one speech
# engine is configured, so without it any import of icespeak fails outright
# with "No voices available". Deployments used to paper over this by setting
# ICESPEAK_KEYS_DIR in .env, which meant a fresh checkout could not run the
# server or the tests without one. Set it unconditionally, and to an absolute
# path, so it holds no matter the working directory.
#
# NB: this must be set before icespeak is first imported. utility is imported
# early by main.py (well before routes, which pulls in icespeak), so importing
# utility is what guarantees the ordering.
os.environ["ICESPEAK_KEYS_DIR"] = str(RESOURCES_DIR)

# Pin the default voice rather than inheriting whatever icespeak's own default
# happens to be. icespeak changed it from Gudrun to Gunnar, which silently
# altered the voice greynir.is answers in: routes/api.py falls back to
# TTS_SETTINGS.DEFAULT_VOICE whenever a request does not name a voice.
#
# tts.py already declares the intended voice per locale, so take it from there
# instead of repeating the name. tts is a leaf module that imports nothing --
# keep it that way, or this becomes an import cycle.
os.environ["ICESPEAK_DEFAULT_VOICE"] = LOCALE_TO_VOICE_ID[DEFAULT_LOCALE]

STATIC_DIR = GREYNIR_ROOT_DIR / "static"
TTS_AUDIO_DIR = STATIC_DIR / "audio" / "tmp"

QUERIES_DIR = GREYNIR_ROOT_DIR / "queries"
QUERIES_GRAMMAR_DIR = QUERIES_DIR / "grammars"
QUERIES_JS_DIR = QUERIES_DIR / "js"
QUERIES_RESOURCES_DIR = QUERIES_DIR / "resources"
QUERIES_UTIL_DIR = QUERIES_DIR / "util"
QUERIES_UTIL_GRAMMAR_DIR = QUERIES_UTIL_DIR / "grammars"
QUERIES_DIALOGUE_DIR = QUERIES_DIR / "dialogues"


@lru_cache(maxsize=32)
def read_txt_api_key(key_name: str, *, folder: Optional[Path] = None) -> str:
    """
    Read the given key from a text file in resources directory. Cached.
    Optionally provide a different path to the folder containing the key file.
    """
    folder = folder or RESOURCES_DIR
    p = folder / f"{key_name}.txt"
    try:
        return p.read_text().strip()
    except FileNotFoundError:
        logging.warning(f"API key file {p} not found in {folder}")
    return ""


@lru_cache(maxsize=32)
def read_json_api_key(
    key_name: str, *, folder: Optional[Path] = None
) -> Dict[str, Any]:
    """
    Read the given key from a JSON file in resources directory. Cached.
    Optionally provide a different path to the folder containing the key file.
    """
    folder = folder or RESOURCES_DIR
    p = folder / f"{key_name}.json"
    try:
        return json.loads(p.read_text())
    except FileNotFoundError:
        logging.warning(f"API key file {p} not found in {folder}")
    return {}


def modules_in_dir(p: Path) -> List[str]:
    """
    Find the import names of all python modules
    in a given directory given a path to a folder
    (can be relative to GREYNIR_ROOT_PATH or absolute).
    """
    p = p.resolve()  # Fully resolve path before working with it
    assert (
        p.exists() and p.is_dir()
    ), f"Directory {str(p)} not found when searching for modules"

    # Return list of python files in
    # import-like format ('.' instead of '/', no '.py')
    return [
        ".".join(pyfile.relative_to(GREYNIR_ROOT_DIR).with_suffix("").parts)
        for pyfile in p.glob("*.py")
        if not pyfile.name.startswith("_")
    ]


def sanitize_filename(fn: str, maxlen: int = 60) -> str:
    """Sanitize a potential filename string by limiting allowed characters."""
    assert maxlen >= 1, "maxlen must be positive"

    ALLOWED_FILE_CHARS = string.ascii_letters + string.digits + "._-"

    # Replace whitespace with underscore
    fn = "_".join([t for t in fn.lower().split()])

    # Rm non-ASCII chars, non-filename chars and limit length
    fn = "".join(c for c in icelandic_asciify(fn) if c in ALLOWED_FILE_CHARS)[
        :maxlen
    ].rstrip("._")

    return fn


def icelandic_asciify(text: str) -> str:
    """Convert Icelandic characters to their ASCII equivalent
    and then remove all non-ASCII characters."""
    if not text:
        return text

    ICECHARS_TO_ASCII = {
        "ð": "d",
        "Ð": "D",
        "á": "a",
        "Á": "A",
        "ú": "u",
        "Ú": "U",
        "í": "i",
        "Í": "I",
        "é": "e",
        "É": "E",
        "þ": "th",
        "Þ": "TH",
        "ó": "o",
        "Ó": "O",
        "ý": "y",
        "Ý": "Y",
        "ö": "o",
        "Ö": "O",
        "æ": "ae",
        "Æ": "AE",
    }

    # Substitute all Icelandic chars for their ASCII equivalents
    t = text
    for k, v in ICECHARS_TO_ASCII.items():
        t = t.replace(k, v)

    # Remove any remaining non-ASCII chars
    t = t.encode("ascii", "ignore").decode()

    return t


def icequote(s: str) -> str:
    """Return string surrounded by Icelandic-style quotation marks."""
    return f"„{s.strip()}“"


def cap_first(s: str) -> str:
    """Capitalize first character in a string."""
    return s[0].upper() + s[1:] if s else s