You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
132 lines
4.2 KiB
132 lines
4.2 KiB
5 months ago
|
from __future__ import annotations
|
||
|
|
||
|
import configparser
|
||
|
import json
|
||
|
import os
|
||
|
import warnings
|
||
|
from typing import Any
|
||
|
|
||
|
conf: dict[str, dict[str, Any]] = {}
|
||
|
default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec")
|
||
|
conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir)
|
||
|
|
||
|
|
||
|
def set_conf_env(conf_dict, envdict=os.environ):
|
||
|
"""Set config values from environment variables
|
||
|
|
||
|
Looks for variables of the form ``FSSPEC_<protocol>`` and
|
||
|
``FSSPEC_<protocol>_<kwarg>``. For ``FSSPEC_<protocol>`` the value is parsed
|
||
|
as a json dictionary and used to ``update`` the config of the
|
||
|
corresponding protocol. For ``FSSPEC_<protocol>_<kwarg>`` there is no
|
||
|
attempt to convert the string value, but the kwarg keys will be lower-cased.
|
||
|
|
||
|
The ``FSSPEC_<protocol>_<kwarg>`` variables are applied after the
|
||
|
``FSSPEC_<protocol>`` ones.
|
||
|
|
||
|
Parameters
|
||
|
----------
|
||
|
conf_dict : dict(str, dict)
|
||
|
This dict will be mutated
|
||
|
envdict : dict-like(str, str)
|
||
|
Source for the values - usually the real environment
|
||
|
"""
|
||
|
kwarg_keys = []
|
||
|
for key in envdict:
|
||
|
if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_":
|
||
|
if key.count("_") > 1:
|
||
|
kwarg_keys.append(key)
|
||
|
continue
|
||
|
try:
|
||
|
value = json.loads(envdict[key])
|
||
|
except json.decoder.JSONDecodeError as ex:
|
||
|
warnings.warn(
|
||
|
f"Ignoring environment variable {key} due to a parse failure: {ex}"
|
||
|
)
|
||
|
else:
|
||
|
if isinstance(value, dict):
|
||
|
_, proto = key.split("_", 1)
|
||
|
conf_dict.setdefault(proto.lower(), {}).update(value)
|
||
|
else:
|
||
|
warnings.warn(
|
||
|
f"Ignoring environment variable {key} due to not being a dict:"
|
||
|
f" {type(value)}"
|
||
|
)
|
||
|
elif key.startswith("FSSPEC"):
|
||
|
warnings.warn(
|
||
|
f"Ignoring environment variable {key} due to having an unexpected name"
|
||
|
)
|
||
|
|
||
|
for key in kwarg_keys:
|
||
|
_, proto, kwarg = key.split("_", 2)
|
||
|
conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key]
|
||
|
|
||
|
|
||
|
def set_conf_files(cdir, conf_dict):
|
||
|
"""Set config values from files
|
||
|
|
||
|
Scans for INI and JSON files in the given dictionary, and uses their
|
||
|
contents to set the config. In case of repeated values, later values
|
||
|
win.
|
||
|
|
||
|
In the case of INI files, all values are strings, and these will not
|
||
|
be converted.
|
||
|
|
||
|
Parameters
|
||
|
----------
|
||
|
cdir : str
|
||
|
Directory to search
|
||
|
conf_dict : dict(str, dict)
|
||
|
This dict will be mutated
|
||
|
"""
|
||
|
if not os.path.isdir(cdir):
|
||
|
return
|
||
|
allfiles = sorted(os.listdir(cdir))
|
||
|
for fn in allfiles:
|
||
|
if fn.endswith(".ini"):
|
||
|
ini = configparser.ConfigParser()
|
||
|
ini.read(os.path.join(cdir, fn))
|
||
|
for key in ini:
|
||
|
if key == "DEFAULT":
|
||
|
continue
|
||
|
conf_dict.setdefault(key, {}).update(dict(ini[key]))
|
||
|
if fn.endswith(".json"):
|
||
|
with open(os.path.join(cdir, fn)) as f:
|
||
|
js = json.load(f)
|
||
|
for key in js:
|
||
|
conf_dict.setdefault(key, {}).update(dict(js[key]))
|
||
|
|
||
|
|
||
|
def apply_config(cls, kwargs, conf_dict=None):
|
||
|
"""Supply default values for kwargs when instantiating class
|
||
|
|
||
|
Augments the passed kwargs, by finding entries in the config dict
|
||
|
which match the classes ``.protocol`` attribute (one or more str)
|
||
|
|
||
|
Parameters
|
||
|
----------
|
||
|
cls : file system implementation
|
||
|
kwargs : dict
|
||
|
conf_dict : dict of dict
|
||
|
Typically this is the global configuration
|
||
|
|
||
|
Returns
|
||
|
-------
|
||
|
dict : the modified set of kwargs
|
||
|
"""
|
||
|
if conf_dict is None:
|
||
|
conf_dict = conf
|
||
|
protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol]
|
||
|
kw = {}
|
||
|
for proto in protos:
|
||
|
# default kwargs from the current state of the config
|
||
|
if proto in conf_dict:
|
||
|
kw.update(conf_dict[proto])
|
||
|
# explicit kwargs always win
|
||
|
kw.update(**kwargs)
|
||
|
kwargs = kw
|
||
|
return kwargs
|
||
|
|
||
|
|
||
|
set_conf_files(conf_dir, conf)
|
||
|
set_conf_env(conf)
|