Skip to content

feat: add functions for project-dataset operations #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 40 commits into from
Apr 8, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
85db1d7
feat(experiment): add JuliaHub._project_datasets
mortenpi Oct 11, 2024
7a4d42b
Merge remote-tracking branch 'origin/main' into mp/project-datasets
mortenpi Feb 27, 2025
878be78
upload WIP
mortenpi Feb 28, 2025
069b44b
end-to-end wip
mortenpi Mar 3, 2025
bae3525
wip docs & cleanup
mortenpi Mar 10, 2025
121b122
cleanup
mortenpi Mar 10, 2025
1f3c1aa
get rid of separate type
mortenpi Mar 11, 2025
f67af70
add basic Dataset constructor tests
mortenpi Mar 12, 2025
9d98a22
make sure Dataset only throws JuliaHubError
mortenpi Mar 12, 2025
25c9f8f
in mocking, .tags is sometimes Any[]
mortenpi Mar 12, 2025
512d230
only silently capture JuliaHubErrors in datasets()
mortenpi Mar 12, 2025
9a81327
add a return
mortenpi Mar 12, 2025
344be60
don't rely on string rep of a parametric type
mortenpi Mar 12, 2025
8f40681
Merge remote-tracking branch 'origin/main' into mp/project-datasets
mortenpi Mar 13, 2025
5e8d69d
Merge branch 'mp/dataset-constructor-tests' into mp/project-datasets
mortenpi Mar 13, 2025
1f906f8
bad merge
mortenpi Mar 13, 2025
8811daa
fix
mortenpi Mar 13, 2025
0590b89
fix current tests
mortenpi Mar 18, 2025
5145e87
fixes & tests for auth
mortenpi Mar 18, 2025
b121c83
Merge remote-tracking branch 'origin/main' into mp/project-datasets
mortenpi Mar 19, 2025
ac47101
add unit tests for listing APIs
mortenpi Mar 19, 2025
52dee3c
basic upload_project_dataset unit tests
mortenpi Mar 19, 2025
3a639f9
Merge branch 'main' into mp/project-datasets
mortenpi Mar 19, 2025
b7ad581
add live tests for projects
mortenpi Mar 20, 2025
5ebf3ce
add option to run tests
mortenpi Mar 20, 2025
6aea1a4
format
mortenpi Mar 24, 2025
cca4d61
docs: use [sources]
mortenpi Mar 24, 2025
b7d86dc
docstrings
mortenpi Mar 24, 2025
bbb6145
auth docs
mortenpi Mar 24, 2025
355856c
changelog
mortenpi Mar 24, 2025
adfd3f5
make JET happy?
mortenpi Mar 24, 2025
f6bcd7c
:facepalm:
mortenpi Mar 24, 2025
c428180
Merge branch 'main' into mp/project-datasets
mortenpi Mar 24, 2025
be1a184
Merge remote-tracking branch 'origin/main' into mp/project-datasets
mortenpi Mar 27, 2025
5aa13fa
fix project_dataset print
mortenpi Mar 27, 2025
e0996f9
Update src/authentication.jl
mortenpi Apr 8, 2025
9d11d5a
don't allow missing in authenticate()
mortenpi Apr 8, 2025
475ab36
fix doctests
mortenpi Apr 8, 2025
a8ced9a
Merge branch 'main' into mp/project-datasets
pfitzseb Apr 8, 2025
4af5e9f
Merge branch 'main' into mp/project-datasets
pfitzseb Apr 8, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixes & tests for auth
  • Loading branch information
mortenpi committed Mar 18, 2025
commit 5145e87920235a11045f80de21c1851badc94893
31 changes: 18 additions & 13 deletions src/authentication.jl
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ mutable struct Authentication
tokenpath::Union{AbstractString, Nothing}=nothing,
email::Union{AbstractString, Nothing}=nothing,
expires::Union{Integer, Nothing}=nothing,
project_uuid::Union{UUIDs.UUID, Nothing}=nothing,
project_id::Union{UUIDs.UUID, Nothing}=nothing,
)
# The authentication() function should take care of sanitizing the inputs here,
# so it is fine to just error() here.
Expand All @@ -61,7 +61,7 @@ mutable struct Authentication
@warn "Invalid auth.toml token path passed to Authentication, ignoring." tokenpath
tokenpath = nothing
end
new(server, username, token, project_uuid, api_version, tokenpath, email, expires)
new(server, username, token, project_id, api_version, tokenpath, email, expires)
end
end

Expand Down Expand Up @@ -232,10 +232,14 @@ This can be set by passing the optional `project` argument, which works as follo
"""
function authenticate end

function authenticate(server::AbstractString, token::Union{AbstractString, Secret})
function authenticate(
server::AbstractString, token::Union{AbstractString, Secret};
project::Union{AbstractString, UUIDs.UUID, Nothing, Missing}=missing,
)
auth = _authentication(
_juliahub_uri(server);
token=isa(token, Secret) ? token : Secret(token),
project_id=_juliahub_project(project),
)
global __AUTH__[] = auth
return auth
Expand All @@ -256,9 +260,9 @@ function authenticate(
),
)
end
project_uuid = _normalize_project(project)
project_id = _juliahub_project(project)
server_uri = _juliahub_uri(server)
auth = Mocking.@mock _authenticate(server_uri; force, maxcount, hook, project_uuid)
auth = Mocking.@mock _authenticate(server_uri; force, maxcount, hook, project_id)
global __AUTH__[] = auth
return auth
end
Expand Down Expand Up @@ -291,14 +295,14 @@ end
function _authenticate(
server_uri::URIs.URI;
force::Bool, maxcount::Integer, hook::Union{Base.Callable, Nothing},
project_uuid::Union{UUID, Nothing},
project_id::Union{UUID, Nothing},
)
isnothing(hook) || PkgAuthentication.register_open_browser_hook(hook)
try
# _authenticate either returns a valid token, or throws
auth_toml = _authenticate_retry(string(server_uri), 1; force, maxcount)
# Note: _authentication may throw, which gets passed on to the user
_authentication(server_uri; project_uuid, auth_toml...)
_authentication(server_uri; project_id, auth_toml...)
finally
isnothing(hook) || PkgAuthentication.clear_open_browser_hook()
end
Expand Down Expand Up @@ -371,7 +375,7 @@ function _authentication(
email::Union{AbstractString, Nothing}=nothing,
username::Union{AbstractString, Nothing}=nothing,
tokenpath::Union{AbstractString, Nothing}=nothing,
project_uuid::Union{UUID, Nothing}=nothing,
project_id::Union{UUID, Nothing}=nothing,
)
# If something goes badly wrong in _get_api_information, it may throw. We won't really
# be able to proceed, since we do not know what JuliaHub APIs to use, so we need to
Expand Down Expand Up @@ -409,12 +413,12 @@ function _authentication(
end
return Authentication(
server, api.api_version, username, token;
email, expires, tokenpath, project_uuid,
email, expires, tokenpath, project_id,
)
end
_authentication(server::AbstractString; kwargs...) = _authentication(URIs.URI(server); kwargs...)

function _normalize_project(
function _juliahub_project(
project::Union{AbstractString, UUIDs.UUID, Nothing, Missing}
)::Union{UUID, Nothing}
if ismissing(project)
Expand All @@ -426,7 +430,7 @@ function _normalize_project(
return project
elseif isa(project, AbstractString)
project_uuid = tryparse(UUIDs.UUID, project)
if isnothing(project)
if isnothing(project_uuid)
throw(
ArgumentError(
"Invalid project_id passed to Authentication() - not a UUID: $(project)"
Expand Down Expand Up @@ -476,7 +480,8 @@ The `force`, `maxcount` and `hook` are relevant for interactive authentication,
same way as in the [`authenticate`](@ref) function.

This is mostly meant to be used to re-acquire authentication tokens in long-running sessions, where
the initial authentication token may have expired.
the initial authentication token may have expired. If the original `auth` object was authenticated
in the context of a project (i.e. `.project_id` is set), the project association will be retained.

As [`Authentication`](@ref) objects are mutable, the token will be updated in all contexts
where the reference to the [`Authentication`](@ref) has been passed to.
Expand Down Expand Up @@ -534,7 +539,7 @@ function reauthenticate!(
end
end
@debug "reauthenticate! -- calling PkgAuthentication" auth.server
new_auth = _authenticate(auth.server; force, maxcount, hook)
new_auth = _authenticate(auth.server; force, maxcount, hook, project_id=auth.project_id)
if new_auth.username != auth.username
throw(
AuthenticationError(
Expand Down
70 changes: 65 additions & 5 deletions test/authentication.jl
Original file line number Diff line number Diff line change
@@ -1,29 +1,70 @@
@testset "_juliahub_project" begin
uuid1 = "80c74bbd-fd5a-4f99-a647-0eec08183ed4"
uuid2 = "24d0f8a7-4c3f-4168-aef4-e49248f3cb40"
withenv("JULIAHUB_PROJECT_UUID" => nothing) do
@test JuliaHub._juliahub_project(uuid1) == UUIDs.UUID(uuid1)
@test_throws ArgumentError JuliaHub._juliahub_project("invalid")
@test JuliaHub._juliahub_project(nothing) === nothing
@test JuliaHub._juliahub_project(missing) === nothing
end
withenv("JULIAHUB_PROJECT_UUID" => uuid1) do
@test JuliaHub._juliahub_project(uuid2) == UUIDs.UUID(uuid2)
@test_throws ArgumentError JuliaHub._juliahub_project("invalid")
@test JuliaHub._juliahub_project(nothing) === nothing
@test JuliaHub._juliahub_project(missing) === UUIDs.UUID(uuid1)
end
end

@testset "JuliaHub.authenticate()" begin
empty!(MOCK_JULIAHUB_STATE)
Mocking.apply(mocking_patch) do
withenv("JULIA_PKG_SERVER" => nothing) do
withenv("JULIA_PKG_SERVER" => nothing, "JULIAHUB_PROJECT_UUID" => nothing) do
@test_throws JuliaHub.AuthenticationError JuliaHub.authenticate()
@test JuliaHub.authenticate("https://juliahub.example.org") isa JuliaHub.Authentication
@test JuliaHub.authenticate("juliahub.example.org") isa JuliaHub.Authentication
end
withenv("JULIA_PKG_SERVER" => "juliahub.example.org") do
withenv("JULIA_PKG_SERVER" => "juliahub.example.org", "JULIAHUB_PROJECT_UUID" => nothing) do
@test JuliaHub.authenticate() isa JuliaHub.Authentication
end
withenv("JULIA_PKG_SERVER" => "https://juliahub.example.org") do
withenv(
"JULIA_PKG_SERVER" => "https://juliahub.example.org", "JULIAHUB_PROJECT_UUID" => nothing
) do
@test JuliaHub.authenticate() isa JuliaHub.Authentication
end
# Conflicting declarations, argument takes precendence
withenv("JULIA_PKG_SERVER" => "https://juliahub-one.example.org") do
# Conflicting declarations, explicit argument takes precedence
withenv(
"JULIA_PKG_SERVER" => "https://juliahub-one.example.org",
"JULIAHUB_PROJECT_UUID" => nothing,
) do
auth = JuliaHub.authenticate("https://juliahub-two.example.org")
@test auth isa JuliaHub.Authentication
@test auth.server == URIs.URI("https://juliahub-two.example.org")
@test auth.project_id === nothing
# check_authentication
MOCK_JULIAHUB_STATE[:invalid_authentication] = false
@test JuliaHub.check_authentication(; auth) === true
MOCK_JULIAHUB_STATE[:invalid_authentication] = true
@test JuliaHub.check_authentication(; auth) === false
delete!(MOCK_JULIAHUB_STATE, :invalid_authentication)
end

# Projects integration
uuid1 = "80c74bbd-fd5a-4f99-a647-0eec08183ed4"
uuid2 = "24d0f8a7-4c3f-4168-aef4-e49248f3cb40"
withenv(
"JULIA_PKG_SERVER" => nothing,
"JULIAHUB_PROJECT_UUID" => uuid1,
) do
auth = JuliaHub.authenticate("https://juliahub.example.org")
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === UUIDs.UUID(uuid1)
auth = JuliaHub.authenticate("https://juliahub.example.org"; project=uuid2)
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === UUIDs.UUID(uuid2)
auth = JuliaHub.authenticate("https://juliahub.example.org"; project=nothing)
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === nothing
end
end
end

Expand Down Expand Up @@ -141,6 +182,25 @@ end
@test a._email === nothing
@test a._expires === nothing
end
# Projects integration
# The JuliaHub.authenticate(server, token) method also takes the `project`
# keyword, and also falls back to the JULIAHUB_PROJECT_UUID.
uuid1 = "80c74bbd-fd5a-4f99-a647-0eec08183ed4"
uuid2 = "24d0f8a7-4c3f-4168-aef4-e49248f3cb40"
withenv(
"JULIA_PKG_SERVER" => nothing,
"JULIAHUB_PROJECT_UUID" => uuid1,
) do
auth = JuliaHub.authenticate(server, token)
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === UUIDs.UUID(uuid1)
auth = JuliaHub.authenticate(server, token; project=uuid2)
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === UUIDs.UUID(uuid2)
auth = JuliaHub.authenticate(server, token; project=nothing)
@test auth.server == URIs.URI("https://juliahub.example.org")
@test auth.project_id === nothing
end
# On old instances, we handle if /api/v1 404s
MOCK_JULIAHUB_STATE[:auth_v1_status] = 404
let a = JuliaHub.authenticate(server, token)
Expand Down
13 changes: 8 additions & 5 deletions test/mocking.jl
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ end

# Set up a mock authentication so that the __auth__() fallbacks would work and use this.
const MOCK_USERNAME = "username"
mockauth(server_uri) = JuliaHub.Authentication(
server_uri, JuliaHub._MISSING_API_VERSION, MOCK_USERNAME, JuliaHub.Secret("")
)
JuliaHub.__AUTH__[] = mockauth(URIs.URI("https://juliahub.com"))
function mockauth(server_uri; project_id, kwargs...)
JuliaHub.Authentication(
server_uri, JuliaHub._MISSING_API_VERSION, MOCK_USERNAME, JuliaHub.Secret("");
project_id,
)
end
JuliaHub.__AUTH__[] = mockauth(URIs.URI("https://juliahub.com"); project_id=nothing)

# The following Mocking.jl patches _rest_request, so the the rest calls would have fixed
# reponses.
Expand Down Expand Up @@ -69,7 +72,7 @@ mocking_patch = [
),
Mocking.@patch(
function JuliaHub._authenticate(server_uri; kwargs...)
return mockauth(server_uri)
return mockauth(server_uri; kwargs...)
end
),
Mocking.@patch(
Expand Down
Loading