Commit 33d3fb35 authored by Paul Gierz's avatar Paul Gierz

Various Updates

* Starts to use typed Python
* Some updates to the docs
* Functions for taking newest and oldest timesteps
parent fdd99166
......@@ -77,7 +77,7 @@ docs: ## generate Sphinx HTML documentation, including API docs
$(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
watchmedo shell-command -p '*.rst; *py' -c '$(MAKE) -C docs html' -R -D .
release: dist ## package and upload a release
twine upload dist/*
......
......@@ -23,7 +23,12 @@ Earth System. It is written in Python and configured with YAML files.
Features
--------
* TODO
* Automatic remapping between various grids via the ``SCRIP`` library
* Ability to run other programs before and after each step
* Configuration via yaml files
* No modification of model source code necessary
Credits
-------
......
......@@ -64,7 +64,7 @@ master_doc = "index"
# General information about the project.
project = u"SCOPE"
copyright = u"2019, Paul Gierz"
copyright = u"2020, Paul Gierz"
author = u"Paul Gierz"
# The version info for the project you're documenting, acts as replacement
......
=====
Usage
=====
=============
Library Usage
=============
An example configuration file is provided under ``examples/scope_config.yaml``:
......@@ -8,6 +8,8 @@ An example configuration file is provided under ``examples/scope_config.yaml``:
:language: yaml
:linenos:
This section describes how to use ``scope`` from a Python program.
To use scope in a project::
import scope
template_replacements:
EXP_ID: "PI_1x10"
DATE_PATTERN: "[0-9]{6}"
scope:
couple_dir: "/work/ollie/pgierz/scope_tests/couple/"
number openMP processes: 8
pism:
type: ice
griddes: ice.griddes
recieve:
solid_earth:
some_variable_name:
interp: bil
some_other_variable_name:
interp: con
transformation:
- expr: "dBdt=...."
send:
solid_earth:
thk:
files:
pattern: "{{ EXP_ID }}_"
take:
what: timesteps
newest: 1
vilma:
type: solid_earth
griddes: n128
recieve:
solid_earth:
thk:
interp: con
send:
ice:
rsl:
files: "{{ EXP_ID }}_vilma_{{ DATE_PATTERN }}.nc"
take:
what: timesteps
newest: 1
......@@ -15,11 +15,20 @@ from scope import Regrid, Preprocess
YAML_AUTO_EXTENSIONS = ["", ".yml", ".yaml", ".YML", ".YAML"]
def yaml_file_to_dict(filepath):
def yaml_file_to_dict(filepath: str) -> dict:
"""
Given a yaml file, returns a corresponding dictionary.
Given a scope configuration yaml file, returns a corresponding dictionary.
If you do not give an extension, tries again after appending one.
If you do not give an extension, tries again after appending one:
+ ``.yml``
+ ``.yaml``
+ ``.YML``
+ ``.YAML``
Note that this function also uses `~jinja2` to replace any templated
variables found in the under the top-level key ``template_replacements``.
This key is then deleted from the remainder of the dictionary.
Parameters
----------
......@@ -30,6 +39,10 @@ def yaml_file_to_dict(filepath):
-------
dict
A dictionary representation of the yaml file.
Raises
------
``OSError`` if the file cannot be found.
"""
for extension in YAML_AUTO_EXTENSIONS:
try:
......@@ -57,15 +70,18 @@ def yaml_file_to_dict(filepath):
@click.version_option()
def main(args=None):
"""Console script for scope."""
click.echo("Replace this message by putting your code into scope.cli.main")
click.echo("See click documentation at http://click.pocoo.org/")
click.echo(
"SCOPE a stand-alone coupler. Please use --help for available operations."
)
click.echo("See documentation at https://scope-coupler.readthedocs.io")
return 0
@main.command()
@click.argument("config_path", type=click.Path(exists=True))
@click.argument("whos_turn")
def regrid(config_path, whos_turn):
def regrid(config_path: str, whos_turn: str) -> None:
"""Command line interface to regridding"""
config = yaml_file_to_dict(config_path)
regridder = Regrid(config, whos_turn)
regridder.regrid()
......@@ -74,7 +90,7 @@ def regrid(config_path, whos_turn):
@main.command()
@click.argument("config_path", type=click.Path(exists=True))
@click.argument("whos_turn")
def preprocess(config_path, whos_turn):
def preprocess(config_path: str, whos_turn: str) -> None:
config = yaml_file_to_dict(config_path)
print(80 * "-")
......
# -*- coding: utf-8 -*-
#!/usr/bin/env python3
"""
Here, the ``scope`` library is described. This allows you to use specific parts
of ``scope`` from other programs.
......@@ -9,7 +9,7 @@ Without a correctly installed ``cdo``, many of these functions/classes will not
work.
Here, we provide a quick summary, but please look at the documentation for each
We provide a quick summary, but please look at the documentation for each
function and class for more complete information. The following functions are
defined:
......@@ -30,6 +30,8 @@ The following classes are defined here:
* ``Regrid`` -- a class to easily regrid from one model to another, depending
on the specifications in the ``scope_config.yaml``
- - - - - -
"""
from functools import wraps
......@@ -41,7 +43,7 @@ import subprocess
import click
def determine_cdo_openMP():
def determine_cdo_openMP() -> bool:
"""
Checks if the ``cdo`` version being used supports ``OpenMP``; useful to
check if you need a ``-P`` flag or not.
......@@ -65,11 +67,81 @@ def determine_cdo_openMP():
return False
class Scope(object):
def __init__(self, config, whos_turn):
"""
Base class for various Scope objects. Other classes should extend this one.
def get_newest_n_timesteps(f: str, take: int) -> str:
"""
Given a file, takes the newest n timesteps for further processing.
Parameters
----------
f : str
The file to use.
take : int
Number of timesteps to take (newest will be taken, i.e. from the end of the file).
Returns
-------
str :
A string with the path to the new file
"""
cdo_command = (
"cdo "
+ " -f nc "
+ " -seltimestep,"
+ str(-take)
+ "/-1 "
+ f
+ " "
+ f.replace(".nc", "_newest_" + str(take) + "_timesteps.nc")
)
click.secho(
"Selecting newest %s timesteps for further processing " % str(take), fg="cyan"
)
click.secho(cdo_command, fg="cyan")
subprocess.run(cdo_command, shell=True, check=True)
return f.replace(".nc", "_newest_" + str(int) + "_timesteps.nc")
def get_oldest_n_timesteps(f: str, take: int) -> str:
"""
Given a file, takes the oldest n timesteps for further processing.
Parameters
----------
f : str
The file to use.
take : int
Number of timesteps to take (newest will be taken, i.e. from the beginning of the file).
Returns
-------
str :
A string with the path to the new file
"""
cdo_command = (
"cdo "
+ " -f nc "
+ " -seltimestep,1/"
+ str(take)
+ " "
+ f
+ " "
+ f.replace(".nc", "_oldest_" + str(take) + "_timesteps.nc")
)
click.secho(
"Selecting oldest %s timesteps for further processing " % str(take), fg="cyan"
)
click.secho(cdo_command, fg="cyan")
subprocess.run(cdo_command, shell=True, check=True)
return f.replace(".nc", "_oldest_" + str(int) + "_timesteps.nc")
class Scope:
"""
Base class for various Scope objects. Other classes should extend this one.
"""
def __init__(self, config: dict, whos_turn: str):
"""
Parameters
----------
config : dict
......@@ -87,6 +159,7 @@ class Scope(object):
permissions to create this folder, the object initialization will
fail...
Some design features are listed below:
* **``pre`` and ``post`` hooks**
......@@ -137,7 +210,7 @@ class Scope(object):
if not os.path.isdir(config["scope"]["couple_dir"]):
os.makedirs(config["scope"]["couple_dir"])
def get_cdo_prefix(self, has_openMP=None):
def get_cdo_prefix(self, has_openMP: bool = False):
"""
Return a string with an appropriate ``cdo`` prefix for using OpenMP
with the ``-P`` flag.
......@@ -161,22 +234,30 @@ class Scope(object):
has_openMP = determine_cdo_openMP()
if has_openMP:
return "cdo -P " + str(self.config["scope"]["number openMP processes"])
else:
return "cdo"
return "cdo"
class ScopeDecorators(object):
class ScopeDecorators:
"""Contains decorators you can use on class methods"""
# FIXME: I don't really like that this is exactly the same above with
# only a positional change. Probably it can abstracted away...
# FIXME: Why is this a static method?
@staticmethod
def _wrap_hook(self, meth):
program_to_call = self.config[self.whos_turn].get("pre_" + meth.__name__, {}).get("program")
program_to_call = (
self.config[self.whos_turn]
.get("pre_" + meth.__name__, {})
.get("program")
)
flags_for_program = self.config[self.whos_turn].get("pre_" + meth.__name__, {}).get("flags_for_program")
flags_for_program = (
self.config[self.whos_turn]
.get("pre_" + meth.__name__, {})
.get("flags_for_program")
)
arguments_for_program = (
self.config[self.whos_turn].get("pre_" + meth.__name__, {}).get("arguments_for_program")
self.config[self.whos_turn]
.get("pre_" + meth.__name__, {})
.get("arguments_for_program")
)
full_process = program_to_call
......@@ -222,9 +303,23 @@ class Preprocess(Scope):
@Scope.ScopeDecorators.pre_hook
@Scope.ScopeDecorators.post_hook
def preprocess(self):
"""
Selects and combines variables from various file into one single file for futher processing.
Files produced:
---------------
* ``<sender_type>_file_for_<reciever_type>`` (e.g. ``atmosphere_file_for_ice.nc``)
Parameters
----------
None
Returns
-------
None
"""
for (sender_type, sender_args) in self._all_senders():
for variable_name, variable_dict in sender_args.items():
logging.debug("The following files will be used for:", variable_name)
self._make_tmp_files_for_variable(variable_name, variable_dict)
self._combine_tmp_variable_files(sender_type)
......@@ -276,6 +371,18 @@ class Preprocess(Scope):
def _construct_filelist(self, var_dict):
"""
Constructs a file list to use for further processing based on user
specifications.
Parameters
----------
var_dict : dict
Configuration dictionary for how to handle one specific variable.
Returns
-------
file_list
A list of files for further processing.
Example
-------
......@@ -283,14 +390,16 @@ class Preprocess(Scope):
* ``files`` may contain:
* a ``filepattern`` in regex to look for
* ``take`` which files to take, either specific, or
* ``take`` which files or timesteps to take, either specific, or
``newest``/``latest`` followed by an integer.
* ``dir`` a directory where to look for the files. Note that if
this is not provided, the default is to fall back to the top level
``outdata_dir`` for the currently sending model.
"""
r = re.compile(var_dict["files"]["pattern"])
file_directory = var_dict["files"].get("dir", self.config[self.whos_turn].get("outdata_dir"))
file_directory = var_dict["files"].get(
"dir", self.config[self.whos_turn].get("outdata_dir")
)
all_files = []
for rootname, _, filenames in os.walk(file_directory):
......@@ -302,17 +411,37 @@ class Preprocess(Scope):
# Just the matching files:
matching_files = sorted([f for f in all_files if r.match(os.path.basename(f))])
if "take" in var_dict["files"]:
if "newest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["newest"]
return matching_files[-take:]
elif "oldest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["oldest"]
return matching_files[:take]
# FIXME: This is wrong:
elif "specific" in var_dict["files"]["take"]:
return var_dict["files"]["take"]["specific"]
else:
return matching_files
if var_dict["take"].get("what") == "files":
if "newest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["newest"]
return matching_files[-take:]
if "oldest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["oldest"]
return matching_files[:take]
# FIXME: This is wrong:
if "specific" in var_dict["files"]["take"]:
return var_dict["files"]["take"]["specific"]
raise SyntaxError("You must specify newest, oldest, or specific!")
if var_dict["take"].get("what") == "timesteps":
if "newest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["newest"]
return get_newest_n_timesteps(matching_files[-1], take)
if "oldest" in var_dict["files"]["take"]:
take = var_dict["files"]["take"]["oldest"]
return get_oldest_n_timesteps(maching_files[-1], take)
if "specific" in var_dict["files"]["take"]:
# return get_specific_timesteps[matching_files[-1], take]
raise NotImplementedError(
"Get specific timesteps not yet implemented!"
)
raise SyntaxError("You must specify newest, oldest, or specific!")
raise SyntaxError(
"""
You specified take in YAML, but didn't specify what to take.
Please either use 'timesteps' or 'files'
"""
)
return matching_files
def _make_tmp_files_for_variable(self, varname, var_dict):
"""
......@@ -349,9 +478,12 @@ class Preprocess(Scope):
"""
flist = self._construct_filelist(var_dict)
logging.debug("The following files will be used for: %s", varname)
for f in flist:
print("- ", f)
code_table = var_dict.get("code table", self.config[self.whos_turn].get("code table"))
logging.debug("- %s", f)
code_table = var_dict.get(
"code table", self.config[self.whos_turn].get("code table")
)
cdo_command = (
self.get_cdo_prefix()
+ " -f nc -t "
......@@ -369,7 +501,9 @@ class Preprocess(Scope):
+ "_file_for_scope.nc"
)
click.secho("Selecting %s for further processing with SCOPE..." % varname, fg="cyan")
click.secho(
"Selecting %s for further processing with SCOPE..." % varname, fg="cyan"
)
click.secho(cdo_command, fg="cyan")
subprocess.run(cdo_command, shell=True, check=True)
......@@ -398,19 +532,29 @@ class Preprocess(Scope):
This executes a ``cdo mergetime`` command to concatenate all files found which
should be sent to particular model.
"""
print(reciever_type)
reciever = self.config.get(self.whos_turn, {}).get("send", {}).get(reciever_type, {})
logging.debug(reciever_type)
reciever = (
self.config.get(self.whos_turn, {}).get("send", {}).get(reciever_type, {})
)
variables_to_send_to_reciever = list(reciever)
files_to_combine = []
for f in os.listdir(self.config["scope"]["couple_dir"]):
fvar = f.replace(self.whos_turn + "_", "").replace("_file_for_scope.nc", "")
if fvar in variables_to_send_to_reciever:
files_to_combine.append(os.path.join(self.config["scope"]["couple_dir"], f))
files_to_combine.append(
os.path.join(self.config["scope"]["couple_dir"], f)
)
output_file = os.path.join(
self.config["scope"]["couple_dir"],
self.config[self.whos_turn]["type"] + "_file_for_" + reciever_type + ".nc",
)
cdo_command = self.get_cdo_prefix() + " mergetime " + " ".join(files_to_combine) + " " + output_file
cdo_command = (
self.get_cdo_prefix()
+ " mergetime "
+ " ".join(files_to_combine)
+ " "
+ output_file
)
click.secho("Combine files for sending to %s" % reciever_type, fg="cyan")
click.secho(cdo_command, fg="cyan")
subprocess.run(cdo_command, shell=True, check=True)
......@@ -419,7 +563,8 @@ class Preprocess(Scope):
class Regrid(Scope):
def _calculate_weights(self, Model, Type, Interp):
regrid_weight_file = os.path.join(
self.config["scope"]["couple_dir"], "_".join([self.config[Model]["type"], Type, Interp, "weight_file.nc"])
self.config["scope"]["couple_dir"],
"_".join([self.config[Model]["type"], Type, Interp, "weight_file.nc"]),
)
cdo_command = (
......@@ -451,10 +596,18 @@ class Regrid(Scope):
if self.config[self.whos_turn].get("recieve"):
for sender_type in self.config[self.whos_turn].get("recieve"):
if self.config[self.whos_turn]["recieve"].get(sender_type):
for Variable in self.config[self.whos_turn]["recieve"].get(sender_type):
for Variable in self.config[self.whos_turn]["recieve"].get(
sender_type
):
Model = self.whos_turn
Type = sender_type
Interp = self.config[self.whos_turn].get("recieve").get(sender_type).get(Variable).get("interp")
Interp = (
self.config[self.whos_turn]
.get("recieve")
.get(sender_type)
.get(Variable)
.get("interp")
)
self.regrid_one_var(Model, Type, Interp, Variable)
def regrid_one_var(self, Model, Type, Interp, Variable):
......@@ -485,3 +638,7 @@ class Regrid(Scope):
click.secho("Remapping: ", fg="cyan")
click.secho(cdo_command, fg="cyan")
subprocess.run(cdo_command, shell=True, check=True)
# -*- coding: utf-8 -*-
# -*- last line -*-
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment