Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions upath/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1236,16 +1236,34 @@ def iterdir(self) -> Iterator[Self]:
fs = base.fs
base_path = base.path
if not fs.isdir(base_path):
raise NotADirectoryError(str(self))
for name in fs.listdir(base_path):
# cached listings can misreport a directory that collides with a
# file of the same name (i.e. s3fs), so check again without cache
fs.invalidate_cache(base_path)
if not fs.isdir(base_path):
raise NotADirectoryError(str(self))
name: str
seen: set[str] = set()
base_name = base_path.removesuffix(sep)
for info in fs.listdir(base_path):
# fsspec returns dictionaries
if isinstance(name, dict):
name = name.get("name")
if isinstance(info, dict):
name = info["name"]
else:
name, info = info, {} # type: ignore[assignment]
name = name.removesuffix(sep)
if name == base_name:
# the directory itself is not a child, i.e. s3 directory
# markers or empty-named prefixes from keys containing "//"
continue
# only want the path name with iterdir
_, _, name = name.rpartition(self.parser.sep)
if name in {".", ".."}:
# Yielding a path object for these makes little sense
continue
# only want the path name with iterdir
_, _, name = name.removesuffix(sep).rpartition(self.parser.sep)
if name in seen:
# a file and a directory can share a name on object stores
continue
seen.add(name)
yield base.with_segments(base_path, name)

def __open_reader__(self) -> BinaryIO:
Expand Down
99 changes: 98 additions & 1 deletion upath/tests/implementations/test_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def s3_with_double_fwd_slash_files(s3_server):
anon, s3so = s3_server
s3 = fsspec.filesystem("s3", anon=False, **s3so)
bucket = "double_fwd_slash_bucket"
s3.mkdir(bucket + "/")
s3.makedirs(bucket, exist_ok=True)
s3.pipe_file(f"{bucket}//key.txt", b"hello world")
try:
yield bucket, anon, s3so
Expand Down Expand Up @@ -226,3 +226,100 @@ def on_collision_rename_file(src, dst):
"src/common_prefix/file1.txt",
"src/common_prefix/file2.txt",
]


@pytest.fixture
def s3_with_explicit_directory_marker(s3_server):
"""issue #226: path.iterdir() yields path itself as the first item (with S3)

Creates a bucket with an explicit directory marker (zero-byte object with
trailing slash) plus files. This simulates folders created via the AWS
Console, which creates actual zero-byte objects as directory markers.

Given objects on S3:
s3://my-bucket/my-directory/ (zero-byte directory marker)
s3://my-bucket/my-directory/0.txt
s3://my-bucket/my-directory/1.txt
"""
import boto3

anon, s3so = s3_server
bucket = "iterdir_issue_226_marker_bucket"
s3_client = boto3.client("s3", **s3so["client_kwargs"])
s3_client.create_bucket(Bucket=bucket)
# Create an explicit directory marker (zero-byte object with trailing slash)
# Use boto3 directly to ensure the trailing slash is preserved in the key
# This is what the AWS Console creates when you "Create folder"
s3_client.put_object(Bucket=bucket, Key="my-directory/", Body=b"")
# Then create files inside the directory
s3_client.put_object(Bucket=bucket, Key="my-directory/0.txt", Body=b"content 0")
s3_client.put_object(Bucket=bucket, Key="my-directory/1.txt", Body=b"content 1")
yield bucket, anon, s3so


@silence_botocore_datetime_deprecation
def test_iterdir_with_explicit_directory_marker__issue_226(
s3_with_explicit_directory_marker,
):
"""issue #226: path.iterdir() yields path itself as the first item (with S3)

See: https://github.com/fsspec/universal_pathlib/issues/226
See: https://medium.com/cyberark-engineering/the-strange-case-of-amazon-s3-bucket-folders-c8d113a8dd01 # noqa: E501
"""
bucket, anon, s3so = s3_with_explicit_directory_marker
directory_path = UPath(f"s3://{bucket}/my-directory", anon=anon, **s3so)

children = list(directory_path.iterdir())

assert directory_path not in children

expected_files = {
directory_path / "0.txt",
directory_path / "1.txt",
}
assert set(children) == expected_files, (
f"iterdir() should yield only the files in the directory. "
f"Expected: {[str(f) for f in expected_files]}, "
f"Got: {[str(c) for c in children]}"
)


@pytest.fixture
def s3_with_file_dir_name_collision(s3_server):
"""an object key that is also the prefix of other objects

Given objects on S3:
s3://my-bucket/my-directory (zero-byte object without trailing slash)
s3://my-bucket/my-directory/0.txt
"""
import boto3

anon, s3so = s3_server
bucket = "iterdir_file_dir_collision_bucket"
s3_client = boto3.client("s3", **s3so["client_kwargs"])
s3_client.create_bucket(Bucket=bucket)
s3_client.put_object(Bucket=bucket, Key="my-directory", Body=b"")
s3_client.put_object(Bucket=bucket, Key="my-directory/0.txt", Body=b"content 0")
yield bucket, anon, s3so


@silence_botocore_datetime_deprecation
def test_iterdir_with_file_dir_name_collision(s3_with_file_dir_name_collision):
bucket, anon, s3so = s3_with_file_dir_name_collision
root = UPath(f"s3://{bucket}/", anon=anon, **s3so)

# the listing contains a file and a directory entry with the same name
children = list(root.iterdir())
assert children == [root / "my-directory"]

# listing the parent populated s3fs' dircache with both entries
assert list(children[0].iterdir()) == [root / "my-directory" / "0.txt"]


@silence_botocore_datetime_deprecation
def test_iterdir_skips_empty_named_prefix(s3_with_double_fwd_slash_files):
"""issue #144: a key starting with "/" creates an empty-named prefix"""
bucket, anon, s3so = s3_with_double_fwd_slash_files
root = UPath(f"s3://{bucket}", anon=anon, **s3so)
# the empty-named child can't be represented as a UPath child of root
assert root.joinpath(bucket) not in list(root.iterdir())
Loading