diff --git a/MANIFEST.in b/MANIFEST.in index 8902170..004ed90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ # This file is part of xrootdpyfs -# Copyright (C) 2015-2023 CERN. +# Copyright (C) 2015-2026 CERN. # # xrootdpyfs is free software; you can redistribute it and/or modify it under the # terms of the Revised BSD License; see LICENSE file for more details. @@ -24,5 +24,7 @@ recursive-include tests *.py recursive-include tests *.html recursive-include tests *.dat recursive-include tests *.txt +recursive-include tests *.py recursive-include .github/workflows *.yml +recursive-include xrootdpyfs *.py include .git-blame-ignore-revs diff --git a/README.rst b/README.rst index 88401d3..362c621 100644 --- a/README.rst +++ b/README.rst @@ -57,15 +57,6 @@ integration: >>> fs.listdir("xrootdpyfs") ['test.txt'] -Or, alternatively using the PyFilesystem opener (note the first -``import xrootdpyfs`` is required to ensure the XRootDPyFS opener is registered): - - >>> import xrootdpyfs - >>> from fs.opener import open_fs - >>> fs = open_fs("root://localhost//tmp/") - >>> fs.listdir("xrootdpyfs") - ['test.txt'] - Reading files: >>> f = fs.open("xrootdpyfs/test.txt") @@ -98,7 +89,8 @@ running XRootD server: .. code-block:: console $ docker build --platform linux/amd64 -t xrootd --progress=plain . - $ docker run --platform linux/amd64 -h xrootdpyfs -it -v :/code xrootd bash + $ docker run --platform linux/amd64 -h xrootdpyfs -it -v :/code xrootd + docker run --platform linux/amd64 -h xrootdpyfs -it -v /Users/nico/workspace/cern/invenio/modules/xrootdpyfs:/code xrootd [xrootdpyfs@xrootdpyfs code]$ xrootd In another shell: diff --git a/docs/index.rst b/docs/index.rst index b8da8ce..f809fff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,12 +24,6 @@ File interface :members: :undoc-members: -Opener ------- -.. automodule:: xrootdpyfs.opener - :members: - :undoc-members: - Environment ----------- .. automodule:: xrootdpyfs.env diff --git a/setup.cfg b/setup.cfg index 4ba7939..d42e991 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ # This file is part of xrootdpyfs -# Copyright (C) 2015-2023 CERN. +# Copyright (C) 2015-2026 CERN. # # xrootdpyfs is free software; you can redistribute it and/or modify it under # the terms of the Revised BSD License; see LICENSE file for more details. @@ -7,7 +7,7 @@ [metadata] name = xrootdpyfs version = attr: xrootdpyfs.__version__ -description = XRootDPyFS is a PyFilesystem interface for XRootD. +description = XRootDPyFS is a PyFilesystem interface for XRootD long_description = file: README.rst, CHANGES.rst keywords = xrootdpyfs license = BSD @@ -25,11 +25,9 @@ classifiers = [options] include_package_data = True packages = find: -python_requires = >=3.6 +python_requires = >=3.9 zip_safe = False install_requires = - fs>=2.0.10,<3.0 - # 5.6.0 breaks compatibility, needs fix xrootd>=5.0.0,<6.0.0 [options.extras_require] @@ -41,11 +39,6 @@ tests = sphinx>=4.5.0 setuptools<82.0.0 -[options.entry_points] -fs.opener = - root = xrootdpyfs.opener:XRootDPyOpener - roots = xrootdpyfs.opener:XRootDPyOpener - [aliases] test = pytest diff --git a/tests/perf.py b/tests/perf.py index 17b7af5..2a05c55 100644 --- a/tests/perf.py +++ b/tests/perf.py @@ -19,14 +19,16 @@ import os import pstats import shutil +import subprocess import tempfile import time from io import StringIO from os.path import join -from fs.opener import open_fs from XRootD import client +from xrootdpyfs.fs import XRootDPyFS + def teardown(tmppath): """Tear down performance test.""" @@ -40,7 +42,10 @@ def setup(): filepath = join(tmppath, filename) # Create test file with random data - os.system("dd bs=1024 count={1} {0}".format(filepath, 1024 * 10)) + subprocess.run( + ["dd", "bs=1024", "count=10240", "if=/dev/urandom", f"of={filepath}"], + check=True, + ) return filename, tmppath, filepath @@ -48,11 +53,10 @@ def setup(): # # Test methods # -def read_pyfs_chunks(url, filename, mode="rb", chunksize=2097152, n=100): +def read_pyfs_chunks(fs, filename, mode="rb", chunksize=2097152, n=100): """Read a file in chunks.""" t1 = time.time() - fs = open_fs(url) assert fs.exists(filename) i = 0 while i < n: @@ -111,12 +115,14 @@ def main(): n = 10 rooturl = "root://localhost/{0}".format(testfilepath) - print("osfs:", testfilepath, read_pyfs_chunks(tmppath, filename, n=n)) - print("pyxrootd:", rooturl, read_pyxrootd_chunks(rooturl, n=n)) + # print("osfs:", testfilepath, read_pyfs_chunks(tmppath, filename, n=n)) + print("pyxrootd:", rooturl, read_pyxrootd_chunks(XRootDPyFS(rooturl), n=n)) pr = profile_start() print( - "xrootdpyfs:", rooturl, read_pyfs_chunks(rooturl, filename, mode="rb-", n=n) + "xrootdpyfs:", + rooturl, + read_pyfs_chunks(XRootDPyFS(rooturl), filename, mode="rb-", n=n), ) profile_end(pr) finally: diff --git a/tests/test_fs.py b/tests/test_fs.py index 6573d07..c795055 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -16,8 +16,12 @@ import pytest from conftest import mkurl -from fs import ResourceType -from fs.errors import ( +from mock import Mock +from XRootD.client.responses import XRootDStatus + +from xrootdpyfs import XRootDPyFile, XRootDPyFS +from xrootdpyfs.fs_utils.enums import ResourceType +from xrootdpyfs.fs_utils.errors import ( DestinationExists, DirectoryNotEmpty, FSError, @@ -29,10 +33,6 @@ ResourceNotFound, Unsupported, ) -from mock import Mock -from XRootD.client.responses import XRootDStatus - -from xrootdpyfs import XRootDPyFile, XRootDPyFS from xrootdpyfs.utils import spliturl diff --git a/tests/test_fs_extras.py b/tests/test_fs_extras.py index acf1c5e..7f2bcc8 100644 --- a/tests/test_fs_extras.py +++ b/tests/test_fs_extras.py @@ -10,9 +10,9 @@ import pytest from conftest import mkurl -from fs.errors import ResourceNotFound from xrootdpyfs import XRootDPyFS +from xrootdpyfs.fs_utils.errors import ResourceNotFound def test_readtext(tmppath): diff --git a/tests/test_opener.py b/tests/test_opener.py deleted file mode 100644 index 93b925f..0000000 --- a/tests/test_opener.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of xrootdpyfs -# Copyright (C) 2015 CERN. -# -# xrootdpyfs is free software; you can redistribute it and/or modify it under -# the terms of the Revised BSD License; see LICENSE file for more details. - -"""Test of XRootDPyOpener.""" - -from conftest import mkurl -from fs.opener import open_fs - - -def test_open_fs_create(tmppath): - """Test open with create.""" - rooturl = mkurl(tmppath) - fs = open_fs(f"{rooturl}/non-existing") - assert fs.listdir("./") - assert not fs.exists("/non-existing") - fs = open_fs(rooturl + "/non-existing", create=True) - assert fs.exists("/non-existing") diff --git a/tests/test_xrdfile.py b/tests/test_xrdfile.py index 585dd6a..439cd6c 100644 --- a/tests/test_xrdfile.py +++ b/tests/test_xrdfile.py @@ -12,16 +12,21 @@ import math from os.path import join -import fs.path import pytest from conftest import mkurl -from fs import Seek -from fs.errors import InvalidPath, PathError, ResourceNotFound, Unsupported -from fs.opener import open_fs from mock import Mock from XRootD.client.responses import XRootDStatus from xrootdpyfs import XRootDPyFile +from xrootdpyfs.fs import XRootDPyFS +from xrootdpyfs.fs_utils.enums import Seek +from xrootdpyfs.fs_utils.errors import ( + InvalidPath, + PathError, + ResourceNotFound, + Unsupported, +) +from xrootdpyfs.fs_utils.path import dirname from xrootdpyfs.utils import is_valid_path, is_valid_url @@ -98,7 +103,7 @@ def get_bin_testfile(tmppath): def get_file(fn, fp, tmppath): path = join(tmppath, fp) fpp = join(path, fn) - fs = open_fs(path) + fs = XRootDPyFS(path) with fs.open(fn) as f: fc = f.read() return {"filename": fn, "dir": fp, "contents": fc, "full_path": fpp} @@ -107,7 +112,7 @@ def get_file(fn, fp, tmppath): def get_file_binary(fn, fp, tmppath): path = join(tmppath, fp) fpp = join(path, fn) - fs = open_fs(path) + fs = XRootDPyFS(path) with fs.open(fn, "rb") as f: fc = f.read() return {"filename": fn, "dir": fp, "contents": fc, "full_path": fpp} @@ -116,14 +121,14 @@ def get_file_binary(fn, fp, tmppath): def copy_file(fn, fp, tmppath): path = join(tmppath, fp) fn_new = fn + "_copy" - this_fs = open_fs(path) + this_fs = XRootDPyFS(path) this_fs.copy(fn, fn_new) return fn_new def get_copy_file(arg, binary=False): # Would get called with e.g. arg=get_tsta_file(...) - fp = fs.path.dirname(arg["full_path"]) + fp = dirname(arg["full_path"]) fn_new = copy_file(arg["filename"], "", fp) return get_file_binary(fn_new, "", fp) if binary else get_file(fn_new, "", fp) @@ -993,7 +998,7 @@ def test_readline(tmppath): fb = get_copy_file(fd) fp, fc = fd["full_path"], fd["contents"] - osfs = open_fs(fs.path.dirname(fd["full_path"])) + osfs = XRootDPyFS(dirname(fd["full_path"])) xfile, pfile = XRootDPyFile(mkurl(fp), "r"), osfs.open(fb["filename"], "r") assert xfile.readline() == pfile.readline().encode() @@ -1104,7 +1109,7 @@ def test_readlines(tmppath): fb = get_copy_file(fd) fp, fc = fd["full_path"], fd["contents"] - osfs = open_fs(fs.path.dirname(fb["full_path"])) + osfs = XRootDPyFS(dirname(fb["full_path"])) xfile, pfile = XRootDPyFile(mkurl(fp), "r"), osfs.open(fb["filename"], "r") xfile.seek(0), pfile.seek(0) diff --git a/xrootdpyfs/__init__.py b/xrootdpyfs/__init__.py index 3a48a40..3ee7735 100644 --- a/xrootdpyfs/__init__.py +++ b/xrootdpyfs/__init__.py @@ -146,16 +146,6 @@ >>> fs.listdir("xrootdpyfs") ['test.txt'] -Or, alternatively using the PyFilesystem opener (note the first -``import xrootdpyfs`` is required to ensure the XRootDPyFS -opener is registered): - - >>> import xrootdpyfs - >>> from fs.opener import open_fs - >>> fs = open_fs("root://localhost//tmp/") - >>> fs.listdir("xrootdpyfs") - ['test.txt'] - Reading files: >>> f = fs.open("xrootdpyfs/test.txt") @@ -183,9 +173,8 @@ """ from .fs import XRootDPyFS -from .opener import XRootDPyOpener from .xrdfile import XRootDPyFile __version__ = "2.0.0" -__all__ = ("__version__", "XRootDPyFS", "XRootDPyOpener", "XRootDPyFile") +__all__ = ("__version__", "XRootDPyFS", "XRootDPyFile") diff --git a/xrootdpyfs/fs.py b/xrootdpyfs/fs.py index 1c68eef..a00821e 100644 --- a/xrootdpyfs/fs.py +++ b/xrootdpyfs/fs.py @@ -20,10 +20,20 @@ import re from glob import fnmatch +from urllib.parse import parse_qs, urlencode -from fs import ResourceType -from fs.base import FS -from fs.errors import ( +from XRootD.client import CopyProcess, FileSystem +from XRootD.client.flags import ( + AccessMode, + DirListFlags, + MkDirFlags, + QueryCode, + StatInfoFlags, +) + +from .fs_utils.base import FS +from .fs_utils.enums import ResourceType +from .fs_utils.errors import ( DestinationExists, DirectoryNotEmpty, FSError, @@ -34,18 +44,17 @@ ResourceNotFound, Unsupported, ) -from fs.info import Info -from fs.path import basename, combine, dirname, frombase, isabs, join, normpath, relpath -from six.moves.urllib.parse import parse_qs, urlencode -from XRootD.client import CopyProcess, FileSystem -from XRootD.client.flags import ( - AccessMode, - DirListFlags, - MkDirFlags, - QueryCode, - StatInfoFlags, +from .fs_utils.info import Info +from .fs_utils.path import ( + basename, + combine, + dirname, + frombase, + isabs, + join, + normpath, + relpath, ) - from .utils import is_valid_path, is_valid_url, spliturl from .xrdfile import XRootDPyFile diff --git a/xrootdpyfs/fs_utils/__init__.py b/xrootdpyfs/fs_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xrootdpyfs/fs_utils/base.py b/xrootdpyfs/fs_utils/base.py new file mode 100644 index 0000000..dd8e296 --- /dev/null +++ b/xrootdpyfs/fs_utils/base.py @@ -0,0 +1,87 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing + +from .walk import BoundWalker, Walker + +if typing.TYPE_CHECKING: + from types import TracebackType + from typing import ( + Dict, + Optional, + Text, + Type, + Union, + ) + + _F = typing.TypeVar("_F", bound="FS") + + +class FS(object): + """Base class for FS objects. + + Copied from PyFileSystem2, which is licensed under the MIT License. + """ + + # This is the "standard" meta namespace. + _meta = {} # type: Dict[Text, Union[Text, int, bool, None]] + + # most FS will use default walking algorithms + walker_class = Walker + + # default to SubFS, used by opendir and should be returned by makedir(s) + subfs_class = None + + def __init__(self): + # type: (...) -> None + """Create a filesystem. See help(type(self)) for accurate signature.""" + self._closed = False + + def __del__(self): + """Auto-close the filesystem on exit.""" + self.close() + + def __enter__(self): + # type: (...) -> FS + """Allow use of filesystem as a context manager.""" + return self + + def __exit__( + self, + exc_type, # type: Optional[Type[BaseException]] + exc_value, # type: Optional[BaseException] + traceback, # type: Optional[TracebackType] + ): + # type: (...) -> None + """Close filesystem on exit.""" + self.close() + + def close(self): + # type: () -> None + """Close the filesystem and release any resources. + + It is important to call this method when you have finished + working with the filesystem. Some filesystems may not finalize + changes until they are closed (archives for example). You may + call this method explicitly (it is safe to call close multiple + times), or you can use the filesystem as a context manager to + automatically close. + + Example: + >>> with OSFS('~/Desktop') as desktop_fs: + ... desktop_fs.writetext( + ... 'note.txt', + ... "Don't forget to tape Game of Thrones" + ... ) + + If you attempt to use a filesystem that has been closed, a + `~fs.errors.FilesystemClosed` exception will be thrown. + + """ + self._closed = True + + @property + def walk(self): + # type: (_F) -> BoundWalker[_F] + """`~fs.walk.BoundWalker`: a walker bound to this filesystem.""" + return self.walker_class.bind(self) diff --git a/xrootdpyfs/fs_utils/enums.py b/xrootdpyfs/fs_utils/enums.py new file mode 100644 index 0000000..e2d6d45 --- /dev/null +++ b/xrootdpyfs/fs_utils/enums.py @@ -0,0 +1,53 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import os +from enum import IntEnum, unique + + +@unique +class ResourceType(IntEnum): + """Resource Types. + + Positive values are reserved, negative values are implementation + dependent. + + Most filesystems will support only directory(1) and file(2). Other + types exist to identify more exotic resource types supported + by Linux filesystems. + + """ + + #: Unknown resource type, used if the filesystem is unable to + #: tell what the resource is. + unknown = 0 + #: A directory. + directory = 1 + #: A simple file. + file = 2 + #: A character file. + character = 3 + #: A block special file. + block_special_file = 4 + #: A first in first out file. + fifo = 5 + #: A socket. + socket = 6 + #: A symlink. + symlink = 7 + + +@unique +class Seek(IntEnum): + """Constants used by `io.IOBase.seek`. + + These match `os.SEEK_CUR`, `os.SEEK_END`, and `os.SEEK_SET` + from the standard library. + + """ + + #: Seek from the current file position. + current = os.SEEK_CUR + #: Seek from the end of the file. + end = os.SEEK_END + #: Seek from the start of the file. + set = os.SEEK_SET diff --git a/xrootdpyfs/fs_utils/errors.py b/xrootdpyfs/fs_utils/errors.py new file mode 100644 index 0000000..4848ecd --- /dev/null +++ b/xrootdpyfs/fs_utils/errors.py @@ -0,0 +1,160 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing + +if typing.TYPE_CHECKING: + from typing import Optional, Text + + +class MissingInfoNamespace(AttributeError): + """An expected namespace is missing.""" + + def __init__(self, namespace): # noqa: D107 + # type: (Text) -> None + self.namespace = namespace + msg = "namespace '{}' is required for this attribute" + super().__init__(msg.format(namespace)) + + def __reduce__(self): + return type(self), (self.namespace,) + + +class FSError(Exception): + """Base exception for the `fs` module.""" + + default_message = "Unspecified error" + + def __init__(self, msg=None): # noqa: D107 + # type: (Optional[Text]) -> None + self._msg = msg or self.default_message + super().__init__() + + def __str__(self): + # type: () -> Text + """Return the error message.""" + msg = self._msg.format(**self.__dict__) + return msg + + def __repr__(self): + # type: () -> Text + msg = self._msg.format(**self.__dict__) + return "{}({!r})".format(self.__class__.__name__, msg) + + +class ResourceError(FSError): + """Base exception class for error associated with a specific resource.""" + + default_message = "failed on path {path}" + + def __init__(self, path, exc=None, msg=None): # noqa: D107 + # type: (Text, Optional[Exception], Optional[Text]) -> None + self.path = path + self.exc = exc + super().__init__(msg=msg) + + def __reduce__(self): + return type(self), (self.path, self.exc, self._msg) + + +class ResourceNotFound(ResourceError): + """Required resource not found.""" + + default_message = "resource '{path}' not found" + + +class DestinationExists(ResourceError): + """Target destination already exists.""" + + default_message = "destination '{path}' exists" + + +class DirectoryNotEmpty(ResourceError): + """Attempt to remove a non-empty directory.""" + + default_message = "directory '{path}' is not empty" + + +class ResourceInvalid(ResourceError): + """Resource has the wrong type.""" + + default_message = "resource '{path}' is invalid for this operation" + + +class PathError(FSError): + """Base exception for errors to do with a path string.""" + + default_message = "path '{path}' is invalid" + + def __init__(self, path, msg=None, exc=None): # noqa: D107 + # type: (Text, Optional[Text], Optional[Exception]) -> None + self.path = path + self.exc = exc + super().__init__(msg=msg) + + def __reduce__(self): + return type(self), (self.path, self._msg, self.exc) + + +class InvalidPath(PathError): + """Path can't be mapped on to the underlaying filesystem.""" + + default_message = "path '{path}' is invalid on this filesystem " + + +class IllegalBackReference(ValueError): + """Too many backrefs exist in a path. + + This error will occur if the back references in a path would be + outside of the root. For example, ``"/foo/../../"``, contains two back + references which would reference a directory above the root. + + Note: + This exception is a subclass of `ValueError` as it is not + strictly speaking an issue with a filesystem or resource. + + """ + + def __init__(self, path): # noqa: D107 + # type: (Text) -> None + self.path = path + msg = ("path '{path}' contains back-references outside of filesystem").format( + path=path + ) + super().__init__(msg) + + def __reduce__(self): + return type(self), (self.path,) + + +class OperationFailed(FSError): + """A specific operation failed.""" + + default_message = "operation failed, {details}" + + def __init__( + self, + path=None, # type: Optional[Text] + exc=None, # type: Optional[Exception] + msg=None, # type: Optional[Text] + ): # noqa: D107 + # type: (...) -> None + self.path = path + self.exc = exc + self.details = "" if exc is None else str(exc) + self.errno = getattr(exc, "errno", None) + super(OperationFailed, self).__init__(msg=msg) + + def __reduce__(self): + return type(self), (self.path, self.exc, self._msg) + + +class RemoteConnectionError(OperationFailed): + """Operations encountered remote connection trouble.""" + + default_message = "remote connection error" + + +class Unsupported(OperationFailed): + """Operation not supported by the filesystem.""" + + default_message = "not supported" diff --git a/xrootdpyfs/fs_utils/info.py b/xrootdpyfs/fs_utils/info.py new file mode 100644 index 0000000..60fba05 --- /dev/null +++ b/xrootdpyfs/fs_utils/info.py @@ -0,0 +1,456 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing +from copy import deepcopy +from datetime import datetime + +from .enums import ResourceType +from .errors import MissingInfoNamespace +from .path import join +from .permissions import Permissions +from .time import epoch_to_datetime + +if typing.TYPE_CHECKING: + from typing import Any, Callable, List, Mapping, Optional, Text, Union, cast + + RawInfo = Mapping[Text, Mapping[Text, object]] + ToDatetime = Callable[[int], datetime] + T = typing.TypeVar("T") + + +class Info(object): + """Container for :ref:`info`. + + Resource information is returned by the following methods: + + * `~fs.base.FS.getinfo` + * `~fs.base.FS.scandir` + * `~fs.base.FS.filterdir` + + Arguments: + raw_info (dict): A dict containing resource info. + to_datetime (callable): A callable that converts an + epoch time to a datetime object. The default uses + `~fs.time.epoch_to_datetime`. + + """ + + __slots__ = ["raw", "_to_datetime", "namespaces"] + + def __init__(self, raw_info, to_datetime=epoch_to_datetime): + # type: (RawInfo, ToDatetime) -> None + """Create a resource info object from a raw info dict.""" + self.raw = raw_info + self._to_datetime = to_datetime + self.namespaces = frozenset(self.raw.keys()) + + def __str__(self): + # type: () -> str + if self.is_dir: + return "".format(self.name) + else: + return "".format(self.name) + + __repr__ = __str__ + + def __eq__(self, other): + # type: (object) -> bool + return self.raw == getattr(other, "raw", None) + + @typing.overload + def _make_datetime(self, t): + # type: (None) -> None + pass + + @typing.overload + def _make_datetime(self, t): # noqa: F811 + # type: (int) -> datetime + pass + + def _make_datetime(self, t): # noqa: F811 + # type: (Optional[int]) -> Optional[datetime] + if t is not None: + return self._to_datetime(t) + else: + return None + + @typing.overload + def get(self, namespace, key): + # type: (Text, Text) -> Any + pass + + @typing.overload # noqa: F811 + def get(self, namespace, key, default): # noqa: F811 + # type: (Text, Text, T) -> Union[Any, T] + pass + + def get(self, namespace, key, default=None): # noqa: F811 + # type: (Text, Text, Optional[Any]) -> Optional[Any] + """Get a raw info value. + + Arguments: + namespace (str): A namespace identifier. + key (str): A key within the namespace. + default (object, optional): A default value to return + if either the namespace or the key within the namespace + is not found. + + Example: + >>> info = my_fs.getinfo("foo.py", namespaces=["details"]) + >>> info.get('details', 'type') + 2 + + """ + try: + return self.raw[namespace].get(key, default) # type: ignore + except KeyError: + return default + + def _require_namespace(self, namespace): + # type: (Text) -> None + """Check if the given namespace is present in the info. + + Raises: + ~fs.errors.MissingInfoNamespace: if the given namespace is not + present in the info. + + """ + if namespace not in self.raw: + raise MissingInfoNamespace(namespace) + + def is_writeable(self, namespace, key): + # type: (Text, Text) -> bool + """Check if a given key in a namespace is writable. + + When creating an `Info` object, you can add a ``_write`` key to + each raw namespace that lists which keys are writable or not. + + In general, this means they are compatible with the `setinfo` + function of filesystem objects. + + Arguments: + namespace (str): A namespace identifier. + key (str): A key within the namespace. + + Returns: + bool: `True` if the key can be modified, `False` otherwise. + + Example: + Create an `Info` object that marks only the ``modified`` key + as writable in the ``details`` namespace:: + + >>> now = time.time() + >>> info = Info({ + ... "basic": {"name": "foo", "is_dir": False}, + ... "details": { + ... "modified": now, + ... "created": now, + ... "_write": ["modified"], + ... } + ... }) + >>> info.is_writeable("details", "created") + False + >>> info.is_writeable("details", "modified") + True + + """ + _writeable = self.get(namespace, "_write", ()) + return key in _writeable + + def has_namespace(self, namespace): + # type: (Text) -> bool + """Check if the resource info contains a given namespace. + + Arguments: + namespace (str): A namespace identifier. + + Returns: + bool: `True` if the namespace was found, `False` otherwise. + + """ + return namespace in self.raw + + def copy(self, to_datetime=None): + # type: (Optional[ToDatetime]) -> Info + """Create a copy of this resource info object.""" + return Info(deepcopy(self.raw), to_datetime=to_datetime or self._to_datetime) + + def make_path(self, dir_path): + # type: (Text) -> Text + """Make a path by joining ``dir_path`` with the resource name. + + Arguments: + dir_path (str): A path to a directory. + + Returns: + str: A path to the resource. + + """ + return join(dir_path, self.name) + + @property + def name(self): + # type: () -> Text + """`str`: the resource name.""" + return cast(Text, self.get("basic", "name")) + + @property + def suffix(self): + # type: () -> Text + """`str`: the last component of the name (with dot). + + In case there is no suffix, an empty string is returned. + + Example: + >>> info = my_fs.getinfo("foo.py") + >>> info.suffix + '.py' + >>> info2 = my_fs.getinfo("bar") + >>> info2.suffix + '' + + """ + name = self.get("basic", "name") + if name.startswith(".") and name.count(".") == 1: + return "" + basename, dot, ext = name.rpartition(".") + return "." + ext if dot else "" + + @property + def suffixes(self): + # type: () -> List[Text] + """`List`: a list of any suffixes in the name. + + Example: + >>> info = my_fs.getinfo("foo.tar.gz") + >>> info.suffixes + ['.tar', '.gz'] + + """ + name = self.get("basic", "name") + if name.startswith(".") and name.count(".") == 1: + return [] + return ["." + suffix for suffix in name.split(".")[1:]] + + @property + def stem(self): + # type: () -> Text + """`str`: the name minus any suffixes. + + Example: + >>> info = my_fs.getinfo("foo.tar.gz") + >>> info.stem + 'foo' + + """ + name = self.get("basic", "name") + if name.startswith("."): + return name + return name.split(".")[0] + + @property + def is_dir(self): + # type: () -> bool + """`bool`: `True` if the resource references a directory.""" + return cast(bool, self.get("basic", "is_dir")) + + @property + def is_file(self): + # type: () -> bool + """`bool`: `True` if the resource references a file.""" + return not cast(bool, self.get("basic", "is_dir")) + + @property + def is_link(self): + # type: () -> bool + """`bool`: `True` if the resource is a symlink.""" + self._require_namespace("link") + return self.get("link", "target", None) is not None + + @property + def type(self): + # type: () -> ResourceType + """`~fs.enums.ResourceType`: the type of the resource. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the 'details' + namespace is not in the Info. + + """ + self._require_namespace("details") + return ResourceType(self.get("details", "type", 0)) + + @property + def accessed(self): + # type: () -> Optional[datetime] + """`~datetime.datetime`: the resource last access time, or `None`. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"details"`` + namespace is not in the Info. + + """ + self._require_namespace("details") + _time = self._make_datetime(self.get("details", "accessed")) + return _time + + @property + def modified(self): + # type: () -> Optional[datetime] + """`~datetime.datetime`: the resource last modification time, or `None`. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"details"`` + namespace is not in the Info. + + """ + self._require_namespace("details") + _time = self._make_datetime(self.get("details", "modified")) + return _time + + @property + def created(self): + # type: () -> Optional[datetime] + """`~datetime.datetime`: the resource creation time, or `None`. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"details"`` + namespace is not in the Info. + + """ + self._require_namespace("details") + _time = self._make_datetime(self.get("details", "created")) + return _time + + @property + def metadata_changed(self): + # type: () -> Optional[datetime] + """`~datetime.datetime`: the resource metadata change time, or `None`. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"details"`` + namespace is not in the Info. + + """ + self._require_namespace("details") + _time = self._make_datetime(self.get("details", "metadata_changed")) + return _time + + @property + def permissions(self): + # type: () -> Optional[Permissions] + """`Permissions`: the permissions of the resource, or `None`. + + Requires the ``"access"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"access"`` + namespace is not in the Info. + + """ + self._require_namespace("access") + _perm_names = self.get("access", "permissions") + if _perm_names is None: + return None + permissions = Permissions(_perm_names) + return permissions + + @property + def size(self): + # type: () -> int + """`int`: the size of the resource, in bytes. + + Requires the ``"details"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"details"`` + namespace is not in the Info. + + """ + self._require_namespace("details") + return cast(int, self.get("details", "size")) + + @property + def user(self): + # type: () -> Optional[Text] + """`str`: the owner of the resource, or `None`. + + Requires the ``"access"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"access"`` + namespace is not in the Info. + + """ + self._require_namespace("access") + return self.get("access", "user") + + @property + def uid(self): + # type: () -> Optional[int] + """`int`: the user id of the resource, or `None`. + + Requires the ``"access"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"access"`` + namespace is not in the Info. + + """ + self._require_namespace("access") + return self.get("access", "uid") + + @property + def group(self): + # type: () -> Optional[Text] + """`str`: the group of the resource owner, or `None`. + + Requires the ``"access"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"access"`` + namespace is not in the Info. + + """ + self._require_namespace("access") + return self.get("access", "group") + + @property + def gid(self): + # type: () -> Optional[int] + """`int`: the group id of the resource, or `None`. + + Requires the ``"access"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"access"`` + namespace is not in the Info. + + """ + self._require_namespace("access") + return self.get("access", "gid") + + @property + def target(self): # noqa: D402 + # type: () -> Optional[Text] + """`str`: the link target (if resource is a symlink), or `None`. + + Requires the ``"link"`` namespace. + + Raises: + ~fs.errors.MissingInfoNamespace: if the ``"link"`` + namespace is not in the Info. + + """ + self._require_namespace("link") + return self.get("link", "target") diff --git a/xrootdpyfs/fs_utils/path.py b/xrootdpyfs/fs_utils/path.py new file mode 100644 index 0000000..3b95b46 --- /dev/null +++ b/xrootdpyfs/fs_utils/path.py @@ -0,0 +1,303 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import re +import typing + +from .errors import IllegalBackReference + +if typing.TYPE_CHECKING: + from typing import List, Text, Tuple + +_requires_normalization = re.compile(r"(^|/)\.\.?($|/)|//", re.UNICODE).search + + +def normpath(path): + # type: (Text) -> Text + """Normalize a path. + + This function simplifies a path by collapsing back-references + and removing duplicated separators. + + Arguments: + path (str): Path to normalize. + + Returns: + str: A valid FS path. + + Example: + >>> normpath("/foo//bar/frob/../baz") + '/foo/bar/baz' + >>> normpath("foo/../../bar") + Traceback (most recent call last): + ... + fs.errors.IllegalBackReference: path 'foo/../../bar' contains back-references outside of filesystem + + """ # noqa: E501 + if path in "/": + return path + + # An early out if there is no need to normalize this path + if not _requires_normalization(path): + return path.rstrip("/") + + prefix = "/" if path.startswith("/") else "" + components = [] # type: List[Text] + try: + for component in path.split("/"): + if component in "..": # True for '..', '.', and '' + if component == "..": + components.pop() + else: + components.append(component) + except IndexError: + # FIXME (@althonos): should be raised from the IndexError + raise IllegalBackReference(path) + return prefix + "/".join(components) + + +def isparent(path1, path2): + # type: (Text, Text) -> bool + """Check if ``path1`` is a parent directory of ``path2``. + + Arguments: + path1 (str): A PyFilesytem path. + path2 (str): A PyFilesytem path. + + Returns: + bool: `True` if ``path1`` is a parent directory of ``path2`` + + Example: + >>> isparent("foo/bar", "foo/bar/spam.txt") + True + >>> isparent("foo/bar/", "foo/bar") + True + >>> isparent("foo/barry", "foo/baz/bar") + False + >>> isparent("foo/bar/baz/", "foo/baz/bar") + False + + """ + bits1 = path1.split("/") + bits2 = path2.split("/") + while bits1 and bits1[-1] == "": + bits1.pop() + if len(bits1) > len(bits2): + return False + for bit1, bit2 in zip(bits1, bits2): + if bit1 != bit2: + return False + return True + + +def frombase(path1, path2): + # type: (Text, Text) -> Text + """Get the final path of ``path2`` that isn't in ``path1``. + + Arguments: + path1 (str): A PyFilesytem path. + path2 (str): A PyFilesytem path. + + Returns: + str: the final part of ``path2``. + + Example: + >>> frombase('foo/bar/', 'foo/bar/baz/egg') + 'baz/egg' + + """ + if not isparent(path1, path2): + raise ValueError("path1 must be a prefix of path2") + return path2[len(path1) :] + + +def isabs(path): + # type: (Text) -> bool + """Check if a path is an absolute path. + + Arguments: + path (str): A PyFilesytem path. + + Returns: + bool: `True` if the path is absolute (starts with a ``'/'``). + + """ + # Somewhat trivial, but helps to make code self-documenting + return path.startswith("/") + + +def abspath(path): + # type: (Text) -> Text + """Convert the given path to an absolute path. + + Since FS objects have no concept of a *current directory*, this + simply adds a leading ``/`` character if the path doesn't already + have one. + + Arguments: + path (str): A PyFilesytem path. + + Returns: + str: An absolute path. + + """ + if not path.startswith("/"): + return "/" + path + return path + + +def relpath(path): + # type: (Text) -> Text + """Convert the given path to a relative path. + + This is the inverse of `abspath`, stripping a leading ``'/'`` from + the path if it is present. + + Arguments: + path (str): A path to adjust. + + Returns: + str: A relative path. + + Example: + >>> relpath('/a/b') + 'a/b' + + """ + return path.lstrip("/") + + +def join(*paths): + # type: (*Text) -> Text + """Join any number of paths together. + + Arguments: + *paths (str): Paths to join, given as positional arguments. + + Returns: + str: The joined path. + + Example: + >>> join('foo', 'bar', 'baz') + 'foo/bar/baz' + >>> join('foo/bar', '../baz') + 'foo/baz' + >>> join('foo/bar', '/baz') + '/baz' + + """ + absolute = False + relpaths = [] # type: List[Text] + for p in paths: + if p: + if p[0] == "/": + del relpaths[:] + absolute = True + relpaths.append(p) + + path = normpath("/".join(relpaths)) + if absolute: + path = abspath(path) + return path + + +def combine(path1, path2): + # type: (Text, Text) -> Text + """Join two paths together. + + This is faster than :func:`~fs.path.join`, but only works when the + second path is relative, and there are no back references in either + path. + + Arguments: + path1 (str): A PyFilesytem path. + path2 (str): A PyFilesytem path. + + Returns: + str: The joint path. + + Example: + >>> combine("foo/bar", "baz") + 'foo/bar/baz' + + """ + if not path1: + return path2.lstrip() + return "{}/{}".format(path1.rstrip("/"), path2.lstrip("/")) + + +def split(path): + # type: (Text) -> Tuple[Text, Text] + """Split a path into (head, tail) pair. + + This function splits a path into a pair (head, tail) where 'tail' is + the last pathname component and 'head' is all preceding components. + + Arguments: + path (str): Path to split + + Returns: + (str, str): a tuple containing the head and the tail of the path. + + Example: + >>> split("foo/bar") + ('foo', 'bar') + >>> split("foo/bar/baz") + ('foo/bar', 'baz') + >>> split("/foo/bar/baz") + ('/foo/bar', 'baz') + + """ + if "/" not in path: + return ("", path) + split = path.rsplit("/", 1) + return (split[0] or "/", split[1]) + + +def dirname(path): + # type: (Text) -> Text + """Return the parent directory of a path. + + This is always equivalent to the 'head' component of the value + returned by ``split(path)``. + + Arguments: + path (str): A PyFilesytem path. + + Returns: + str: the parent directory of the given path. + + Example: + >>> dirname('foo/bar/baz') + 'foo/bar' + >>> dirname('/foo/bar') + '/foo' + >>> dirname('/foo') + '/' + + """ + return split(path)[0] + + +def basename(path): + # type: (Text) -> Text + """Return the basename of the resource referenced by a path. + + This is always equivalent to the 'tail' component of the value + returned by split(path). + + Arguments: + path (str): A PyFilesytem path. + + Returns: + str: the name of the resource at the given path. + + Example: + >>> basename('foo/bar/baz') + 'baz' + >>> basename('foo/bar') + 'bar' + >>> basename('foo/bar/') + '' + + """ + return split(path)[1] diff --git a/xrootdpyfs/fs_utils/permissions.py b/xrootdpyfs/fs_utils/permissions.py new file mode 100644 index 0000000..0004dd8 --- /dev/null +++ b/xrootdpyfs/fs_utils/permissions.py @@ -0,0 +1,302 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing + +if typing.TYPE_CHECKING: + from typing import Iterable, Iterator, List, Optional, Text, Tuple, Type, Union + + +class _PermProperty(object): + """Creates simple properties to get/set permissions.""" + + def __init__(self, name): + # type: (Text) -> None + self._name = name + self.__doc__ = "Boolean for '{}' permission.".format(name) + + def __get__(self, obj, obj_type=None): + # type: (Permissions, Optional[Type[Permissions]]) -> bool + return self._name in obj + + def __set__(self, obj, value): + # type: (Permissions, bool) -> None + if value: + obj.add(self._name) + else: + obj.remove(self._name) + + +class Permissions(object): + """An abstraction for file system permissions. + + Permissions objects store information regarding the permissions + on a resource. It supports Linux permissions, but is generic enough + to manage permission information from almost any filesystem. + + Example: + >>> from xrootdpyfs.fs_permissions import Permissions + >>> p = Permissions(user='rwx', group='rw-', other='r--') + >>> print(p) + rwxrw-r-- + >>> p.mode + 500 + >>> oct(p.mode) + '0o764' + + """ + + _LINUX_PERMS = [ + ("setuid", 2048), + ("setguid", 1024), + ("sticky", 512), + ("u_r", 256), + ("u_w", 128), + ("u_x", 64), + ("g_r", 32), + ("g_w", 16), + ("g_x", 8), + ("o_r", 4), + ("o_w", 2), + ("o_x", 1), + ] # type: List[Tuple[Text, int]] + _LINUX_PERMS_NAMES = [_name for _name, _mask in _LINUX_PERMS] # type: List[Text] + + def __init__( + self, + names=None, # type: Optional[Iterable[Text]] + mode=None, # type: Optional[int] + user=None, # type: Optional[Text] + group=None, # type: Optional[Text] + other=None, # type: Optional[Text] + sticky=None, # type: Optional[bool] + setuid=None, # type: Optional[bool] + setguid=None, # type: Optional[bool] + ): + # type: (...) -> None + """Create a new `Permissions` instance. + + Arguments: + names (list, optional): A list of permissions. + mode (int, optional): A mode integer. + user (str, optional): A triplet of *user* permissions, e.g. + ``"rwx"`` or ``"r--"`` + group (str, optional): A triplet of *group* permissions, e.g. + ``"rwx"`` or ``"r--"`` + other (str, optional): A triplet of *other* permissions, e.g. + ``"rwx"`` or ``"r--"`` + sticky (bool, optional): A boolean for the *sticky* bit. + setuid (bool, optional): A boolean for the *setuid* bit. + setguid (bool, optional): A boolean for the *setguid* bit. + + """ + if names is not None: + self._perms = set(names) + elif mode is not None: + self._perms = {name for name, mask in self._LINUX_PERMS if mode & mask} + else: + perms = self._perms = set() + perms.update("u_" + p for p in user or "" if p != "-") + perms.update("g_" + p for p in group or "" if p != "-") + perms.update("o_" + p for p in other or "" if p != "-") + + if sticky: + self._perms.add("sticky") + if setuid: + self._perms.add("setuid") + if setguid: + self._perms.add("setguid") + + def __repr__(self): + # type: () -> Text + if not self._perms.issubset(self._LINUX_PERMS_NAMES): + _perms_str = ", ".join("'{}'".format(p) for p in sorted(self._perms)) + return "Permissions(names=[{}])".format(_perms_str) + + def _check(perm, name): + # type: (Text, Text) -> Text + return name if perm in self._perms else "" + + user = "".join((_check("u_r", "r"), _check("u_w", "w"), _check("u_x", "x"))) + group = "".join((_check("g_r", "r"), _check("g_w", "w"), _check("g_x", "x"))) + other = "".join((_check("o_r", "r"), _check("o_w", "w"), _check("o_x", "x"))) + args = [] + _fmt = "user='{}', group='{}', other='{}'" + basic = _fmt.format(user, group, other) + args.append(basic) + if self.sticky: + args.append("sticky=True") + if self.setuid: + args.append("setuid=True") + if self.setuid: + args.append("setguid=True") + return "Permissions({})".format(", ".join(args)) + + def __str__(self): + # type: () -> Text + return self.as_str() + + def __iter__(self): + # type: () -> Iterator[Text] + return iter(self._perms) + + def __contains__(self, permission): + # type: (object) -> bool + return permission in self._perms + + def __eq__(self, other): + # type: (object) -> bool + if isinstance(other, Permissions): + names = other.dump() # type: object + else: + names = other + return self.dump() == names + + def __ne__(self, other): + # type: (object) -> bool + return not self.__eq__(other) + + @classmethod + def parse(cls, ls): + # type: (Text) -> Permissions + """Parse permissions in Linux notation.""" + user = ls[:3] + group = ls[3:6] + other = ls[6:9] + return cls(user=user, group=group, other=other) + + @classmethod + def load(cls, permissions): + # type: (List[Text]) -> Permissions + """Load a serialized permissions object.""" + return cls(names=permissions) + + @classmethod + def create(cls, init=None): + # type: (Union[int, Iterable[Text], None]) -> Permissions + """Create a permissions object from an initial value. + + Arguments: + init (int or list, optional): May be None to use `0o777` + permissions, a mode integer, or a list of permission names. + + Returns: + int: mode integer that may be used for instance by `os.makedir`. + + Example: + >>> Permissions.create(None) + Permissions(user='rwx', group='rwx', other='rwx') + >>> Permissions.create(0o700) + Permissions(user='rwx', group='', other='') + >>> Permissions.create(['u_r', 'u_w', 'u_x']) + Permissions(user='rwx', group='', other='') + + """ + if init is None: + return cls(mode=0o777) + if isinstance(init, cls): + return init + if isinstance(init, int): + return cls(mode=init) + if isinstance(init, list): + return cls(names=init) + raise ValueError("permissions is invalid") + + @classmethod + def get_mode(cls, init): + # type: (Union[int, Iterable[Text], None]) -> int + """Convert an initial value to a mode integer.""" + return cls.create(init).mode + + def copy(self): + # type: () -> Permissions + """Make a copy of this permissions object.""" + return Permissions(names=list(self._perms)) + + def dump(self): + # type: () -> List[Text] + """Get a list suitable for serialization.""" + return sorted(self._perms) + + def as_str(self): + # type: () -> Text + """Get a Linux-style string representation of permissions.""" + perms = [ + c if name in self._perms else "-" + for name, c in zip(self._LINUX_PERMS_NAMES[-9:], "rwxrwxrwx") + ] + if "setuid" in self._perms: + perms[2] = "s" if "u_x" in self._perms else "S" + if "setguid" in self._perms: + perms[5] = "s" if "g_x" in self._perms else "S" + if "sticky" in self._perms: + perms[8] = "t" if "o_x" in self._perms else "T" + + perm_str = "".join(perms) + return perm_str + + @property + def mode(self): + # type: () -> int + """`int`: mode integer.""" + mode = 0 + for name, mask in self._LINUX_PERMS: + if name in self._perms: + mode |= mask + return mode + + @mode.setter + def mode(self, mode): + # type: (int) -> None + self._perms = {name for name, mask in self._LINUX_PERMS if mode & mask} + + u_r = _PermProperty("u_r") + u_w = _PermProperty("u_w") + u_x = _PermProperty("u_x") + + g_r = _PermProperty("g_r") + g_w = _PermProperty("g_w") + g_x = _PermProperty("g_x") + + o_r = _PermProperty("o_r") + o_w = _PermProperty("o_w") + o_x = _PermProperty("o_x") + + sticky = _PermProperty("sticky") + setuid = _PermProperty("setuid") + setguid = _PermProperty("setguid") + + def add(self, *permissions): + # type: (*Text) -> None + """Add permission(s). + + Arguments: + *permissions (str): Permission name(s), such as ``'u_w'`` + or ``'u_x'``. + + """ + self._perms.update(permissions) + + def remove(self, *permissions): + # type: (*Text) -> None + """Remove permission(s). + + Arguments: + *permissions (str): Permission name(s), such as ``'u_w'`` + or ``'u_x'``.s + + """ + self._perms.difference_update(permissions) + + def check(self, *permissions): + # type: (*Text) -> bool + """Check if one or more permissions are enabled. + + Arguments: + *permissions (str): Permission name(s), such as ``'u_w'`` + or ``'u_x'``. + + Returns: + bool: `True` if all given permissions are set. + + """ + return self._perms.issuperset(permissions) diff --git a/xrootdpyfs/fs_utils/time.py b/xrootdpyfs/fs_utils/time.py new file mode 100644 index 0000000..8f87c53 --- /dev/null +++ b/xrootdpyfs/fs_utils/time.py @@ -0,0 +1,27 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing +from datetime import datetime, timezone + +if typing.TYPE_CHECKING: + from typing import Optional + + +@typing.overload +def epoch_to_datetime(t): # noqa: D103 + # type: (None) -> None + pass + + +@typing.overload +def epoch_to_datetime(t): # noqa: D103 + # type: (int) -> datetime + pass + + +def epoch_to_datetime(t): + # type: (Optional[int]) -> Optional[datetime] + """Convert epoch time to a UTC datetime.""" + if t is None: + return None + return datetime.fromtimestamp(t, tz=timezone.utc) diff --git a/xrootdpyfs/fs_utils/walk.py b/xrootdpyfs/fs_utils/walk.py new file mode 100644 index 0000000..af566a2 --- /dev/null +++ b/xrootdpyfs/fs_utils/walk.py @@ -0,0 +1,794 @@ +"""Copied from PyFileSystem2, which is licensed under the MIT License.""" + +import typing +from collections import defaultdict, deque, namedtuple + +from .errors import FSError +from .info import Info +from .path import abspath, combine, normpath + +if typing.TYPE_CHECKING: + from typing import ( + Any, + Callable, + Collection, + Iterator, + List, + MutableMapping, + Optional, + Text, + Tuple, + Type, + ) + + from .base import FS + + OnError = Callable[[Text, Exception], bool] + +_F = typing.TypeVar("_F", bound="FS") + + +Step = namedtuple("Step", "path, dirs, files") +"""type: a *step* in a directory walk. +""" + + +def make_repr(class_name, *args, **kwargs): + # type: (Text, *object, **Tuple[object, object]) -> Text + """Generate a repr string. + + Positional arguments should be the positional arguments used to + construct the class. Keyword arguments should consist of tuples of + the attribute value and default. If the value is the default, then + it won't be rendered in the output. + + Example: + >>> class MyClass(object): + ... def __init__(self, name=None): + ... self.name = name + ... def __repr__(self): + ... return make_repr('MyClass', 'foo', name=(self.name, None)) + >>> MyClass('Will') + MyClass('foo', name='Will') + >>> MyClass(None) + MyClass('foo') + + """ + arguments = [repr(arg) for arg in args] + arguments.extend( + [ + "{}={!r}".format(name, value) + for name, (value, default) in sorted(kwargs.items()) + if value != default + ] + ) + return "{}({})".format(class_name, ", ".join(arguments)) + + +class Walker(object): + """A walker object recursively lists directories in a filesystem.""" + + def __init__( + self, + ignore_errors=False, # type: bool + on_error=None, # type: Optional[OnError] + search="breadth", # type: Text + filter=None, # type: Optional[List[Text]] + exclude=None, # type: Optional[List[Text]] + filter_dirs=None, # type: Optional[List[Text]] + exclude_dirs=None, # type: Optional[List[Text]] + max_depth=None, # type: Optional[int] + filter_glob=None, # type: Optional[List[Text]] + exclude_glob=None, # type: Optional[List[Text]] + ): + # type: (...) -> None + """Create a new `Walker` instance. + + Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will + be raised. + on_error (callable, optional): If ``ignore_errors`` is `False`, + then this callable will be invoked for a path and the + exception object. It should return `True` to ignore the error, + or `False` to re-raise it. + search (str): If ``"breadth"`` then the directory will be + walked *top down*. Set to ``"depth"`` to walk *bottom up*. + filter (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``["*.py"]``. Files will + only be returned if the final component matches one of the + patterns. + exclude (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``["~*"]``. Files matching + any of these patterns will be removed from the walk. + filter_dirs (list, optional): A list of patterns that will be used + to match directories names. The walk will only open directories + that match at least one of these patterns. Directories will + only be returned if the final component matches one of the + patterns. + exclude_dirs (list, optional): A list of patterns that will be + used to filter out directories from the walk. e.g. + ``['*.svn', '*.git']``. Directory names matching any of these + patterns will be removed from the walk. + max_depth (int, optional): Maximum directory depth to walk. + filter_glob (list, optional): If supplied, this parameter + should be a list of path patterns e.g. ``["foo/**/*.py"]``. + Resources will only be returned if their global path or + an extension of it matches one of the patterns. + exclude_glob (list, optional): If supplied, this parameter + should be a list of path patterns e.g. ``["foo/**/*.pyc"]``. + Resources will not be returned if their global path or + an extension of it matches one of the patterns. + + """ + if search not in ("breadth", "depth"): + raise ValueError("search must be 'breadth' or 'depth'") + self.ignore_errors = ignore_errors + if on_error: + if ignore_errors: + raise ValueError("on_error is invalid when ignore_errors==True") + else: + on_error = self._ignore_errors if ignore_errors else self._raise_errors + if not callable(on_error): + raise TypeError("on_error must be callable") + + self.on_error = on_error + self.search = search + self.filter = filter + self.exclude = exclude + self.filter_dirs = filter_dirs + self.exclude_dirs = exclude_dirs + self.filter_glob = filter_glob + self.exclude_glob = exclude_glob + self.max_depth = max_depth + super(Walker, self).__init__() + + @classmethod + def _ignore_errors(cls, path, error): + # type: (Text, Exception) -> bool + """Ignore dir scan errors when called.""" + return True + + @classmethod + def _raise_errors(cls, path, error): + # type: (Text, Exception) -> bool + """Re-raise dir scan errors when called.""" + return False + + @classmethod + def _calculate_depth(cls, path): + # type: (Text) -> int + """Calculate the 'depth' of a directory path (i.e. count components).""" + _path = path.strip("/") + return _path.count("/") + 1 if _path else 0 + + @classmethod + def bind(cls, fs): + # type: (_F) -> BoundWalker[_F] + """Bind a `Walker` instance to a given filesystem. + + This *binds* in instance of the Walker to a given filesystem, so + that you won't need to explicitly provide the filesystem as a + parameter. + + Arguments: + fs (FS): A filesystem object. + + Returns: + ~fs.walk.BoundWalker: a bound walker. + + Examples: + Use this method to explicitly bind a filesystem instance:: + + >>> walker = Walker.bind(my_fs) + >>> for path in walker.files(filter=['*.py']): + ... print(path) + /foo.py + /bar.py + + Unless you have written a customized walker class, you will + be unlikely to need to call this explicitly, as filesystem + objects already have a ``walk`` attribute which is a bound + walker object:: + + >>> for path in my_fs.walk.files(filter=['*.py']): + ... print(path) + /foo.py + /bar.py + + """ + return BoundWalker(fs) + + def __repr__(self): + # type: () -> Text + return make_repr( + self.__class__.__name__, + ignore_errors=(self.ignore_errors, False), + on_error=(self.on_error, None), + search=(self.search, "breadth"), + filter=(self.filter, None), + exclude=(self.exclude, None), + filter_dirs=(self.filter_dirs, None), + exclude_dirs=(self.exclude_dirs, None), + max_depth=(self.max_depth, None), + filter_glob=(self.filter_glob, None), + exclude_glob=(self.exclude_glob, None), + ) + + def _iter_walk( + self, + fs, # type: FS + path, # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Tuple[Text, Optional[Info]]] + """Get the walk generator.""" + if self.search == "breadth": + return self._walk_breadth(fs, path, namespaces=namespaces) + else: + return self._walk_depth(fs, path, namespaces=namespaces) + + def _check_open_dir(self, fs, path, info): + # type: (FS, Text, Info) -> bool + """Check if a directory should be considered in the walk.""" + full_path = combine(path, info.name) + if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name): + return False + if self.exclude_glob is not None and fs.match_glob( + self.exclude_glob, full_path + ): + return False + if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name): + return False + if self.filter_glob is not None and not fs.match_glob( + self.filter_glob, full_path, accept_prefix=True + ): + return False + return self.check_open_dir(fs, path, info) + + def check_open_dir(self, fs, path, info): + # type: (FS, Text, Info) -> bool + """Check if a directory should be opened. + + Override to exclude directories from the walk. + + Arguments: + fs (FS): A filesystem instance. + path (str): Path to directory. + info (Info): A resource info object for the directory. + + Returns: + bool: `True` if the directory should be opened. + + """ + return True + + def _check_scan_dir(self, fs, path, info, depth): + # type: (FS, Text, Info, int) -> bool + """Check if a directory contents should be scanned.""" + if self.max_depth is not None and depth >= self.max_depth: + return False + return self.check_scan_dir(fs, path, info) + + def check_scan_dir(self, fs, path, info): + # type: (FS, Text, Info) -> bool + """Check if a directory should be scanned. + + Override to omit scanning of certain directories. If a directory + is omitted, it will appear in the walk but its files and + sub-directories will not. + + Arguments: + fs (FS): A filesystem instance. + path (str): Path to directory. + info (Info): A resource info object for the directory. + + Returns: + bool: `True` if the directory should be scanned. + + """ + return True + + def _check_file(self, fs, dir_path, info): + # type: (FS, Text, Info) -> bool + """Check if a filename should be included.""" + # Weird check required for backwards compatibility, + # when _check_file did not exist. + if Walker._check_file == type(self)._check_file: + if self.exclude is not None and fs.match(self.exclude, info.name): + return False + if self.exclude_glob is not None and fs.match_glob( + self.exclude_glob, dir_path + "/" + info.name + ): + return False + if self.filter is not None and not fs.match(self.filter, info.name): + return False + if self.filter_glob is not None and not fs.match_glob( + self.filter_glob, dir_path + "/" + info.name, accept_prefix=True + ): + return False + return self.check_file(fs, info) + + def check_file(self, fs, info): + # type: (FS, Info) -> bool + """Check if a filename should be included. + + Override to exclude files from the walk. + + Arguments: + fs (FS): A filesystem instance. + info (Info): A resource info object. + + Returns: + bool: `True` if the file should be included. + + """ + return True + + def _scan( + self, + fs, # type: FS + dir_path, # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Info] + """Get an iterator of `Info` objects for a directory path. + + Arguments: + fs (FS): A filesystem instance. + dir_path (str): A path to a directory on the filesystem. + namespaces (list): A list of additional namespaces to + include in the `Info` objects. + + Returns: + ~collections.Iterator: iterator of `Info` objects for + resources within the given path. + + """ + try: + for info in fs.scandir(dir_path, namespaces=namespaces): + yield info + except FSError as error: + if not self.on_error(dir_path, error): + raise + + def walk( + self, + fs, # type: FS + path="/", # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Step] + """Walk the directory structure of a filesystem. + + Arguments: + fs (FS): A filesystem instance. + path (str): A path to a directory on the filesystem. + namespaces (list, optional): A list of additional namespaces + to add to the `Info` objects. + + Returns: + collections.Iterator: an iterator of `~fs.walk.Step` instances. + + The return value is an iterator of ``(, , )`` + named tuples, where ```` is an absolute path to a + directory, and ```` and ```` are a list of + `~fs.info.Info` objects for directories and files in ````. + + Example: + >>> walker = Walker(filter=['*.py']) + >>> for path, dirs, files in walker.walk(my_fs, namespaces=["details"]): + ... print("[{}]".format(path)) + ... print("{} directories".format(len(dirs))) + ... total = sum(info.size for info in files) + ... print("{} bytes".format(total)) + [/] + 2 directories + 55 bytes + ... + + """ + _path = abspath(normpath(path)) + dir_info = defaultdict(list) # type: MutableMapping[Text, List[Info]] + _walk = self._iter_walk(fs, _path, namespaces=namespaces) + for dir_path, info in _walk: + if info is None: + dirs = [] # type: List[Info] + files = [] # type: List[Info] + for _info in dir_info[dir_path]: + (dirs if _info.is_dir else files).append(_info) + yield Step(dir_path, dirs, files) + del dir_info[dir_path] + else: + dir_info[dir_path].append(info) + + def files(self, fs, path="/"): + # type: (FS, Text) -> Iterator[Text] + """Walk a filesystem, yielding absolute paths to files. + + Arguments: + fs (FS): A filesystem instance. + path (str): A path to a directory on the filesystem. + + Yields: + str: absolute path to files on the filesystem found + recursively within the given directory. + + """ + _combine = combine + for _path, info in self._iter_walk(fs, path=path): + if info is not None and not info.is_dir: + yield _combine(_path, info.name) + + def dirs(self, fs, path="/"): + # type: (FS, Text) -> Iterator[Text] + """Walk a filesystem, yielding absolute paths to directories. + + Arguments: + fs (FS): A filesystem instance. + path (str): A path to a directory on the filesystem. + + Yields: + str: absolute path to directories on the filesystem found + recursively within the given directory. + + """ + _combine = combine + for _path, info in self._iter_walk(fs, path=path): + if info is not None and info.is_dir: + yield _combine(_path, info.name) + + def info( + self, + fs, # type: FS + path="/", # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Tuple[Text, Info]] + """Walk a filesystem, yielding tuples of ``(, )``. + + Arguments: + fs (FS): A filesystem instance. + path (str): A path to a directory on the filesystem. + namespaces (list, optional): A list of additional namespaces + to add to the `Info` objects. + + Yields: + (str, Info): a tuple of ``(, )``. + + """ + _combine = combine + _walk = self._iter_walk(fs, path=path, namespaces=namespaces) + for _path, info in _walk: + if info is not None: + yield _combine(_path, info.name), info + + def _walk_breadth( + self, + fs, # type: FS + path, # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Tuple[Text, Optional[Info]]] + """Walk files using a *breadth first* search.""" + queue = deque([path]) + push = queue.appendleft + pop = queue.pop + + _combine = combine + _scan = self._scan + _calculate_depth = self._calculate_depth + _check_open_dir = self._check_open_dir + _check_scan_dir = self._check_scan_dir + _check_file = self._check_file + + depth = _calculate_depth(path) + + while queue: + dir_path = pop() + for info in _scan(fs, dir_path, namespaces=namespaces): + if info.is_dir: + _depth = _calculate_depth(dir_path) - depth + 1 + if _check_open_dir(fs, dir_path, info): + yield dir_path, info # Opened a directory + if _check_scan_dir(fs, dir_path, info, _depth): + push(_combine(dir_path, info.name)) + else: + if _check_file(fs, dir_path, info): + yield dir_path, info # Found a file + yield dir_path, None # End of directory + + def _walk_depth( + self, + fs, # type: FS + path, # type: Text + namespaces=None, # type: Optional[Collection[Text]] + ): + # type: (...) -> Iterator[Tuple[Text, Optional[Info]]] + """Walk files using a *depth first* search.""" + # No recursion! + + _combine = combine + _scan = self._scan + _calculate_depth = self._calculate_depth + _check_open_dir = self._check_open_dir + _check_scan_dir = self._check_scan_dir + _check_file = self._check_file + depth = _calculate_depth(path) + + stack = [ + (path, _scan(fs, path, namespaces=namespaces), None) + ] # type: List[Tuple[Text, Iterator[Info], Optional[Tuple[Text, Info]]]] + + push = stack.append + + while stack: + dir_path, iter_files, parent = stack[-1] + info = next(iter_files, None) + if info is None: + if parent is not None: + yield parent + yield dir_path, None + del stack[-1] + elif info.is_dir: + _depth = _calculate_depth(dir_path) - depth + 1 + if _check_open_dir(fs, dir_path, info): + if _check_scan_dir(fs, dir_path, info, _depth): + _path = _combine(dir_path, info.name) + push( + ( + _path, + _scan(fs, _path, namespaces=namespaces), + (dir_path, info), + ) + ) + else: + yield dir_path, info + else: + if _check_file(fs, dir_path, info): + yield dir_path, info + + +class BoundWalker(typing.Generic[_F]): + """A class that binds a `Walker` instance to a `FS` instance. + + You will typically not need to create instances of this class + explicitly. Filesystems have a `~FS.walk` property which returns a + `BoundWalker` object. + + Example: + >>> tmp_fs = fs.tempfs.TempFS() + >>> tmp_fs.walk + BoundWalker(TempFS()) + + A `BoundWalker` is callable. Calling it is an alias for the + `~fs.walk.BoundWalker.walk` method. + + """ + + def __init__(self, fs, walker_class=Walker): + # type: (_F, Type[Walker]) -> None + """Create a new walker bound to the given filesystem. + + Arguments: + fs (FS): A filesystem instance. + walker_class (type): A `~fs.walk.WalkerBase` + sub-class. The default uses `~fs.walk.Walker`. + + """ + self.fs = fs + self.walker_class = walker_class + + def __repr__(self): + # type: () -> Text + return "BoundWalker({!r})".format(self.fs) + + def _make_walker(self, *args, **kwargs): + # type: (*Any, **Any) -> Walker + """Create a walker instance.""" + walker = self.walker_class(*args, **kwargs) + return walker + + def walk( + self, + path="/", # type: Text + namespaces=None, # type: Optional[Collection[Text]] + **kwargs # type: Any + ): + # type: (...) -> Iterator[Step] + """Walk the directory structure of a filesystem. + + Arguments: + path (str): + namespaces (list, optional): A list of namespaces to include + in the resource information, e.g. ``['basic', 'access']`` + (defaults to ``['basic']``). + + Keyword Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will be + raised. + on_error (callable): If ``ignore_errors`` is `False`, then + this callable will be invoked with a path and the exception + object. It should return `True` to ignore the error, or + `False` to re-raise it. + search (str): If ``'breadth'`` then the directory will be + walked *top down*. Set to ``'depth'`` to walk *bottom up*. + filter (list): If supplied, this parameter should be a list + of file name patterns, e.g. ``['*.py']``. Files will only be + returned if the final component matches one of the + patterns. + exclude (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``['~*', '.*']``. Files matching + any of these patterns will be removed from the walk. + filter_dirs (list, optional): A list of patterns that will be used + to match directories paths. The walk will only open directories + that match at least one of these patterns. + exclude_dirs (list): A list of patterns that will be used + to filter out directories from the walk, e.g. ``['*.svn', + '*.git']``. + max_depth (int, optional): Maximum directory depth to walk. + + Returns: + ~collections.Iterator: an iterator of ``(, , )`` + named tuples, where ```` is an absolute path to a + directory, and ```` and ```` are a list of + `~fs.info.Info` objects for directories and files in ````. + + Example: + >>> walker = Walker(filter=['*.py']) + >>> for path, dirs, files in walker.walk(my_fs, namespaces=['details']): + ... print("[{}]".format(path)) + ... print("{} directories".format(len(dirs))) + ... total = sum(info.size for info in files) + ... print("{} bytes".format(total)) + [/] + 2 directories + 55 bytes + ... + + This method invokes `Walker.walk` with bound `FS` object. + + """ + walker = self._make_walker(**kwargs) + return walker.walk(self.fs, path=path, namespaces=namespaces) + + __call__ = walk + + def files(self, path="/", **kwargs): + # type: (Text, **Any) -> Iterator[Text] + """Walk a filesystem, yielding absolute paths to files. + + Arguments: + path (str): A path to a directory. + + Keyword Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will be + raised. + on_error (callable): If ``ignore_errors`` is `False`, then + this callable will be invoked with a path and the exception + object. It should return `True` to ignore the error, or + `False` to re-raise it. + search (str): If ``'breadth'`` then the directory will be + walked *top down*. Set to ``'depth'`` to walk *bottom up*. + filter (list): If supplied, this parameter should be a list + of file name patterns, e.g. ``['*.py']``. Files will only be + returned if the final component matches one of the + patterns. + exclude (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``['~*', '.*']``. Files matching + any of these patterns will be removed from the walk. + filter_dirs (list, optional): A list of patterns that will be used + to match directories paths. The walk will only open directories + that match at least one of these patterns. + exclude_dirs (list): A list of patterns that will be used + to filter out directories from the walk, e.g. ``['*.svn', + '*.git']``. + max_depth (int, optional): Maximum directory depth to walk. + + Returns: + ~collections.Iterator: An iterator over file paths (absolute + from the filesystem root). + + This method invokes `Walker.files` with the bound `FS` object. + + """ + walker = self._make_walker(**kwargs) + return walker.files(self.fs, path=path) + + def dirs(self, path="/", **kwargs): + # type: (Text, **Any) -> Iterator[Text] + """Walk a filesystem, yielding absolute paths to directories. + + Arguments: + path (str): A path to a directory. + + Keyword Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will be + raised. + on_error (callable): If ``ignore_errors`` is `False`, then + this callable will be invoked with a path and the exception + object. It should return `True` to ignore the error, or + `False` to re-raise it. + search (str): If ``'breadth'`` then the directory will be + walked *top down*. Set to ``'depth'`` to walk *bottom up*. + filter_dirs (list, optional): A list of patterns that will be used + to match directories paths. The walk will only open directories + that match at least one of these patterns. + exclude_dirs (list): A list of patterns that will be used + to filter out directories from the walk, e.g. ``['*.svn', + '*.git']``. + max_depth (int, optional): Maximum directory depth to walk. + + Returns: + ~collections.Iterator: an iterator over directory paths + (absolute from the filesystem root). + + This method invokes `Walker.dirs` with the bound `FS` object. + + """ + walker = self._make_walker(**kwargs) + return walker.dirs(self.fs, path=path) + + def info( + self, + path="/", # type: Text + namespaces=None, # type: Optional[Collection[Text]] + **kwargs # type: Any + ): + # type: (...) -> Iterator[Tuple[Text, Info]] + """Walk a filesystem, yielding path and `Info` of resources. + + Arguments: + path (str): A path to a directory. + namespaces (list, optional): A list of namespaces to include + in the resource information, e.g. ``['basic', 'access']`` + (defaults to ``['basic']``). + + Keyword Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will be + raised. + on_error (callable): If ``ignore_errors`` is `False`, then + this callable will be invoked with a path and the exception + object. It should return `True` to ignore the error, or + `False` to re-raise it. + search (str): If ``'breadth'`` then the directory will be + walked *top down*. Set to ``'depth'`` to walk *bottom up*. + filter (list): If supplied, this parameter should be a list + of file name patterns, e.g. ``['*.py']``. Files will only be + returned if the final component matches one of the + patterns. + exclude (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``['~*', '.*']``. Files matching + any of these patterns will be removed from the walk. + filter_dirs (list, optional): A list of patterns that will be used + to match directories paths. The walk will only open directories + that match at least one of these patterns. + exclude_dirs (list): A list of patterns that will be used + to filter out directories from the walk, e.g. ``['*.svn', + '*.git']``. + max_depth (int, optional): Maximum directory depth to walk. + + Returns: + ~collections.Iterable: an iterable yielding tuples of + ``(, )``. + + This method invokes `Walker.info` with the bound `FS` object. + + """ + walker = self._make_walker(**kwargs) + return walker.info(self.fs, path=path, namespaces=namespaces) + + +# Allow access to default walker from the module +# For example: +# fs.walk.walk_files() + +default_walker = Walker() +walk = default_walker.walk +walk_files = default_walker.files +walk_info = default_walker.info +walk_dirs = default_walker.dirs diff --git a/xrootdpyfs/opener.py b/xrootdpyfs/opener.py deleted file mode 100644 index 2f64ac9..0000000 --- a/xrootdpyfs/opener.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of xrootdpyfs -# Copyright (C) 2015, 2016 CERN. -# -# xrootdpyfs is free software; you can redistribute it and/or modify it under -# the terms of the Revised BSD License; see LICENSE file for more details. - -"""PyFilesystem opener for XRootD.""" - -from fs.opener import Opener -from fs.path import split - -from .fs import XRootDPyFS -from .utils import spliturl - - -class XRootDPyOpener(Opener): - """XRootD PyFilesystem Opener.""" - - protocols = ["root", "roots"] - - def open_fs(self, fs_url, parse_result, writeable, create, cwd): - """Open a filesystem object from a FS URL. - - Arguments: - fs_url (str): A filesystem URL. - parse_result (~fs.opener.parse.ParseResult): A parsed filesystem URL. - writeable (bool): `True` if the filesystem must be writable. - create (bool): `True` if the filesystem should be created if it does not exist. - cwd (str): The current working directory (generally only relevant for OS filesystems). - - Raises: - fs.opener.errors.OpenerError: If a filesystem could not be opened for any reason. - - Returns: - `~fs.base.FS`: A filesystem instance. - """ - root_url, path, query = spliturl(fs_url) - - dirpath, _ = split(path) - - fs = XRootDPyFS(root_url + dirpath + query) - - if create and path: - fs.makedir(path, recursive=True, allow_recreate=True) - - if dirpath: - fs = fs.opendir(dirpath) - - return fs diff --git a/xrootdpyfs/utils.py b/xrootdpyfs/utils.py index c5b5088..8a0dde3 100644 --- a/xrootdpyfs/utils.py +++ b/xrootdpyfs/utils.py @@ -9,8 +9,8 @@ """Helper methods for working with root URLs.""" import re +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse from XRootD.client import URL from XRootD.client.flags import OpenFlags diff --git a/xrootdpyfs/xrdfile.py b/xrootdpyfs/xrdfile.py index de51bfe..42c5c01 100644 --- a/xrootdpyfs/xrdfile.py +++ b/xrootdpyfs/xrdfile.py @@ -10,12 +10,17 @@ import sys -from fs import Seek -from fs.errors import InvalidPath, PathError, ResourceNotFound, Unsupported -from fs.path import basename -from six import b, binary_type, text_type from XRootD.client import File +from xrootdpyfs.fs_utils.enums import Seek +from xrootdpyfs.fs_utils.errors import ( + InvalidPath, + PathError, + ResourceNotFound, + Unsupported, +) + +from .fs_utils.path import basename from .utils import is_valid_path, is_valid_url, spliturl, translate_file_mode_to_flags @@ -113,8 +118,8 @@ def __init__( self._ipp = 0 self._size = -1 self._iterator = None - self._newline = newline or b("\n") - self._buffer = b("") + self._newline = newline or b"\n" + self._buffer = b"" self._buffer_pos = 0 # flag translation @@ -223,7 +228,7 @@ def readline(self): A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). """ - bits = [self._buffer if self._buffer_pos == self.tell() else b("")] + bits = [self._buffer if self._buffer_pos == self.tell() else b""] indx = bits[-1].find(self._newline) if indx == -1: @@ -236,7 +241,7 @@ def readline(self): indx = bit.find(self._newline) if indx == -1: - return b("").join(bits) + return b"".join(bits) indx += len(self._newline) extra = bits[-1][indx:] @@ -245,7 +250,7 @@ def readline(self): self._buffer = extra self._buffer_pos = self.tell() - return b("").join(bits) + return bytes("").join(bits) def readlines(self): """Read until EOF using readline(). @@ -279,10 +284,10 @@ def write(self, data, flushing=False): if "a" in self.mode: self.seek(0, Seek.end) - if not isinstance(data, binary_type): + if not isinstance(data, bytes): if isinstance(data, bytearray): data = bytes(data) - elif isinstance(data, text_type): + elif isinstance(data, str): data = data.encode(self.encoding, self.errors) statmsg, res = self._file.write(data, offset=self._ipp)