Merge pull request #48 from hborla/fix-subdirectory-extraction

Skip subdirectories under `proposals/` in the network extraction job.

Author: shahmishal

Sources/EvolutionMetadataExtraction/ExtractionJob.swift

@@ -95,11 +95,17 @@ extension ExtractionJob {
         async let previousResults = ignorePreviousResults ? [] : PreviousResultsFetcher.fetchPreviousResults()
 
         let mainBranchInfo = try await GitHubFetcher.fetchMainBranch()
-        async let proposalContentItems = GitHubFetcher.fetchProposalContentItems(for: mainBranchInfo.commit.sha)
-        
-        let proposalSpecs = try await proposalContentItems.enumerated().map { $1.proposalSpec(sortIndex: $0) }
-        
-        return await ExtractionJob(source: source, output: output, branchInfo: mainBranchInfo, proposalListing: try proposalContentItems, proposalSpecs: proposalSpecs, previousResults: try await previousResults, expectedResults: nil, forcedExtractionIDs: forcedExtractionIDs, toolVersion: toolVersion, extractionDate: extractionDate)
+        let proposalContentItems = try await GitHubFetcher.fetchProposalContentItems(for: mainBranchInfo.commit.sha)
+
+        // The proposals/ directory may have subdirectories for
+        // proposals from specific workgroups. For now, proposals
+        // in those subdirectories are filtered out of this proposal
+        // specs array.
+        let proposalSpecs = proposalContentItems.enumerated().compactMap {
+          $1.proposalSpec(sortIndex: $0)
+        }
+
+        return ExtractionJob(source: source, output: output, branchInfo: mainBranchInfo, proposalListing: proposalContentItems, proposalSpecs: proposalSpecs, previousResults: try await previousResults, expectedResults: nil, forcedExtractionIDs: forcedExtractionIDs, toolVersion: toolVersion, extractionDate: extractionDate)
     }
 
     private static func makeSnapshotExtractionJob(source: Source, output: Output, ignorePreviousResults: Bool, forcedExtractionIDs: [String], toolVersion: String, extractionDate: Date) throws -> ExtractionJob {

Sources/EvolutionMetadataExtraction/Utilities/Networking.swift

@@ -54,7 +54,7 @@ struct GitHubContentItem: Codable {
     var url: String
     var html_url: String
     var git_url: String
-    var download_url: String
+    var download_url: String?
     var type: String
     var _links: LinkContainer
 
@@ -63,9 +63,12 @@ struct GitHubContentItem: Codable {
         var git: String
         var html: String
     }
-    
-    func proposalSpec(sortIndex: Int) ->  ProposalSpec {
-        ProposalSpec(url: URL(string: download_url)!, sha: sha, sortIndex: sortIndex)
+
+    /// Returns `nil` if this content item corresponds to a
+    /// subdirectory instead of a direct proposal document.
+    func proposalSpec(sortIndex: Int) ->  ProposalSpec? {
+        guard let download_url else { return nil }
+        return ProposalSpec(url: URL(string: download_url)!, sha: sha, sortIndex: sortIndex)
     }
 }