Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 43 additions & 43 deletions pins/boards.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from __future__ import annotations

import functools
import inspect
import logging
Expand All @@ -7,7 +9,7 @@
from datetime import datetime, timedelta
from io import IOBase
from pathlib import Path
from typing import Mapping, Optional, Protocol, Sequence
from typing import Mapping, Protocol, Sequence

from importlib_resources import files

Expand All @@ -23,7 +25,7 @@


class IFileSystem(Protocol):
protocol: "str | list"
protocol: str | list

def ls(self, path: str) -> Sequence[str]: ...

Expand All @@ -47,11 +49,11 @@ class BaseBoard:

def __init__(
self,
board: "str | Path",
board: str | Path,
fs: IFileSystem,
versioned=True,
meta_factory=MetaFactory(),
allow_pickle_read: "bool | None" = None,
allow_pickle_read: bool | None = None,
):
self.board = str(board)
self.fs = fs
Expand Down Expand Up @@ -170,7 +172,7 @@ def pin_list(self):

return [name for name in pin_names if name not in self.reserved_pin_names]

def pin_fetch(self, name: str, version: Optional[str] = None) -> Meta:
def pin_fetch(self, name: str, version: str | None = None) -> Meta:
meta = self.pin_meta(name, version)

# TODO: sanity check caching (since R pins does a cache touch here)
Expand All @@ -182,7 +184,7 @@ def pin_fetch(self, name: str, version: Optional[str] = None) -> Meta:
# so they could pin_fetch and then examine the result, a la pin_download
return meta

def pin_read(self, name, version: Optional[str] = None, hash: Optional[str] = None):
def pin_read(self, name, version: str | None = None, hash: str | None = None):
"""Return the data stored in a pin.

Parameters
Expand Down Expand Up @@ -216,13 +218,13 @@ def pin_read(self, name, version: Optional[str] = None, hash: Optional[str] = No
def _pin_store(
self,
x,
name: Optional[str] = None,
type: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
metadata: Optional[Mapping] = None,
versioned: Optional[bool] = None,
created: Optional[datetime] = None,
name: str | None = None,
type: str | None = None,
title: str | None = None,
description: str | None = None,
metadata: Mapping | None = None,
versioned: bool | None = None,
created: datetime | None = None,
) -> Meta:
if type == "feather":
warn_deprecated(
Expand Down Expand Up @@ -301,13 +303,13 @@ def _pin_store(
def pin_write(
self,
x,
name: Optional[str] = None,
type: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
metadata: Optional[Mapping] = None,
versioned: Optional[bool] = None,
created: Optional[datetime] = None,
name: str | None = None,
type: str | None = None,
title: str | None = None,
description: str | None = None,
metadata: Mapping | None = None,
versioned: bool | None = None,
created: datetime | None = None,
) -> Meta:
"""Write a pin object to the board.

Expand Down Expand Up @@ -386,7 +388,7 @@ def pin_download(self, name, version=None, hash=None) -> Sequence[str]:

def pin_upload(
self,
paths: "str | list[str]",
paths: str | list[str],
name=None,
title=None,
description=None,
Expand Down Expand Up @@ -440,7 +442,7 @@ def pin_version_delete(self, name: str, version: str):
pin_version_path = self.construct_path([pin_name, version])
self.fs.rm(pin_version_path, recursive=True)

def pin_versions_prune(self, name, n: "int | None" = None, days: "int | None" = None):
def pin_versions_prune(self, name, n: int | None = None, days: int | None = None):
"""Delete old versions of a pin.

Parameters
Expand Down Expand Up @@ -534,7 +536,7 @@ def pin_search(self, search=None, as_df=True):
# looks with meta objects in it.
return res

def pin_delete(self, names: "str | Sequence[str]"):
def pin_delete(self, names: str | Sequence[str]):
"""Delete a pin (or pins), removing it from the board.

Parameters
Expand Down Expand Up @@ -611,14 +613,14 @@ def prepare_pin_version(
self,
pin_dir_path,
x,
name: Optional[str] = None,
type: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
metadata: Optional[Mapping] = None,
versioned: Optional[bool] = None,
created: Optional[datetime] = None,
object_name: Optional[str] = None,
name: str | None = None,
type: str | None = None,
title: str | None = None,
description: str | None = None,
metadata: Mapping | None = None,
versioned: bool | None = None,
created: datetime | None = None,
object_name: str | None = None,
):
meta = self._create_meta(
pin_dir_path,
Expand All @@ -642,14 +644,14 @@ def _create_meta(
self,
pin_dir_path,
x,
name: Optional[str] = None,
type: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
metadata: Optional[Mapping] = None,
versioned: Optional[bool] = None,
created: Optional[datetime] = None,
object_name: Optional[str] = None,
name: str | None = None,
type: str | None = None,
title: str | None = None,
description: str | None = None,
metadata: Mapping | None = None,
versioned: bool | None = None,
created: datetime | None = None,
object_name: str | None = None,
):
if name is None:
raise NotImplementedError("Name must be specified.")
Expand Down Expand Up @@ -933,9 +935,7 @@ def pin_list(self):
return names

@ExtendMethodDoc
def pin_write(
self, *args, access_type=None, versioned: Optional[bool] = None, **kwargs
):
def pin_write(self, *args, access_type=None, versioned: bool | None = None, **kwargs):
"""Write a pin.

Extends parent method in the following ways:
Expand Down Expand Up @@ -1121,7 +1121,7 @@ def path_to_deploy_version(self, name: str, version: str):
def user_name(self):
return self.fs.api.get_user()["username"]

def prepare_pin_version(self, pin_dir_path, x, name: "str | None", *args, **kwargs):
def prepare_pin_version(self, pin_dir_path, x, name: str | None, *args, **kwargs):
# RSC pin names can have form <user_name>/<name>, but this will try to
# create the object in a directory named <user_name>. So we grab just
# the <name> part.
Expand Down
36 changes: 19 additions & 17 deletions pins/meta.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from __future__ import annotations

from dataclasses import InitVar, asdict, dataclass, field, fields
from pathlib import Path
from typing import ClassVar, List, Mapping, Optional, Sequence, Union
from typing import ClassVar, Mapping, Sequence

import yaml

Expand All @@ -24,7 +26,7 @@ class MetaRaw:
The type of pin data stored. This is used to determine how to read / write it.
"""

file: "str | Sequence[str] | None"
file: str | Sequence[str] | None
type: str
name: str

Expand Down Expand Up @@ -63,16 +65,16 @@ class Meta:

"""

_excluded: ClassVar["set[str]"] = {"name", "version", "local"}
_excluded: ClassVar[set[str]] = {"name", "version", "local"}

title: Optional[str]
description: Optional[str]
title: str | None
description: str | None

# TODO(defer): different from R pins, which has a local field
created: str
pin_hash: str

file: Union[str, Sequence[str]]
file: str | Sequence[str]
file_size: int
type: str

Expand All @@ -84,14 +86,14 @@ class Meta:
# pin_hash, created, etc.."
version: VersionRaw

tags: Optional[List[str]] = None
name: Optional[str] = None
tags: list[str] | None = None
name: str | None = None
user: Mapping = field(default_factory=dict)
local: Mapping = field(default_factory=dict)

unknown_fields: InitVar["dict | None"] = None
unknown_fields: InitVar[dict | None] = None

def __post_init__(self, unknown_fields: "dict | None"):
def __post_init__(self, unknown_fields: dict | None):
unknown_fields = {} if unknown_fields is None else unknown_fields

self._unknown_fields = unknown_fields
Expand Down Expand Up @@ -119,7 +121,7 @@ def to_pin_dict(self):
return d

@classmethod
def from_pin_dict(cls, data, pin_name, version, local=None) -> "Meta":
def from_pin_dict(cls, data, pin_name, version, local=None) -> Meta:
# TODO: re-arrange Meta argument positions to reflect what's been
# learned about default arguments. e.g. title was not used at some
# point in api_version 1
Expand All @@ -142,17 +144,17 @@ def from_pin_dict(cls, data, pin_name, version, local=None) -> "Meta":
unknown_fields=unknown,
)

def to_pin_yaml(self, f: Optional[IOBase] = None) -> "str | None":
def to_pin_yaml(self, f: IOBase | None = None) -> str | None:
data = self.to_pin_dict()

return yaml.dump(data, f)


@dataclass
class MetaV0:
file: Union[str, Sequence[str]]
file: str | Sequence[str]
type: str
description: "str | None"
description: str | None

name: str

Expand All @@ -173,7 +175,7 @@ def to_dict(self):
return asdict(self)

@classmethod
def from_pin_dict(cls, data, pin_name, version, local=None) -> "MetaV0":
def from_pin_dict(cls, data, pin_name, version, local=None) -> MetaV0:
# could infer from dataclasses.fields(), but seems excessive.
req_fields = {"type", "description"}

Expand Down Expand Up @@ -211,7 +213,7 @@ def get_version_for_meta(self, api_version) -> Version:

def create(
self,
base_folder: "str | Path",
base_folder: str | Path,
files: Sequence[StrOrFile],
type,
# TODO: when files is a string name should be okay as None
Expand Down Expand Up @@ -266,7 +268,7 @@ def read_pin_yaml(
self,
f: IOBase,
pin_name: str,
version: "str | VersionRaw",
version: str | VersionRaw,
local=None,
) -> Meta:
if isinstance(version, str):
Expand Down
10 changes: 6 additions & 4 deletions pins/versions.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from __future__ import annotations

import logging
from dataclasses import asdict, dataclass
from datetime import datetime
from typing import Mapping, Sequence, Union
from typing import Mapping, Sequence

from xxhash import xxh64

Expand Down Expand Up @@ -64,7 +66,7 @@ def hash_file(f: IOBase, block_size: int = -1) -> str:
return hasher.hexdigest()

@classmethod
def from_string(cls, version: str) -> "Version":
def from_string(cls, version: str) -> Version:
parts = version.split("-")

if len(parts) != 2:
Expand Down Expand Up @@ -93,8 +95,8 @@ def from_string(cls, version: str) -> "Version":

@classmethod
def from_files(
cls, files: Sequence[StrOrFile], created: Union[datetime, None] = None
) -> "Version":
cls, files: Sequence[StrOrFile], created: datetime | None = None
) -> Version:
hashes = []
for f in files:
hash_ = cls.hash_file(open(f, "rb") if isinstance(f, str) else f)
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ extend-exclude = ["docs"]
select = [
"E", # Style
"F", # Errors
"FA", # Use from __future__ import annotations for cleaner type hints
"I", # Import sorting
"UP", # Upgrade to latest supported Python syntax
"W", # Style
Expand Down