Add support for the @availability annotations

Author: sethmlarson

.github/validate-pr/index.js

@@ -60,7 +60,7 @@ async function run () {
     }
   }
 
-  const specFiles = files.filter(file => file.includes('specification'))
+  const specFiles = files.filter(file => file.includes('specification') && !file.includes('compiler/test'))
   const table = []
 
   cd(tsValidationPath)

compiler/src/model/build-model.ts

@@ -68,6 +68,15 @@ export function compileEndpoints (): Record<string, model.Endpoint> {
       name: api,
       description: spec.documentation.description,
       docUrl: spec.documentation.url,
+      // Setting these values by default should be removed
+      // when we no longer use rest-api-spec stubs as the
+      // source of truth for stability/visibility.
+      availability: {
+        stack: {
+          stability: spec.stability,
+          visibility: spec.visibility
+        }
+      },
       stability: spec.stability,
       visibility: spec.visibility,
       request: null,

compiler/src/model/metamodel.ts

@@ -369,7 +369,7 @@ export enum Stability {
 }
 export enum Visibility {
   public = 'public',
-  featureFlag = 'feature_flag',
+  feature_flag = 'feature_flag',
   private = 'private'
 }
 
@@ -378,12 +378,25 @@ export class Deprecation {
   description: string
 }
 
+export class Availabilities {
+  stack?: Availability
+  serverless?: Availability
+}
+
+export class Availability {
+  since?: string
+  featureFlag?: string
+  stability?: Stability
+  visibility?: Visibility
+}
+
 export class Endpoint {
   name: string
   description: string
   docUrl: string
   docId?: string
   deprecation?: Deprecation
+  availability: Availabilities
 
   /**
    * If the request value is `null` it means that there is not yet a

compiler/src/model/utils.ts

@@ -575,7 +575,7 @@ export function hoistRequestAnnotations (
   request: model.Request, jsDocs: JSDoc[], mappings: Record<string, model.Endpoint>, response: model.TypeName | null
 ): void {
   const knownRequestAnnotations = [
-    'since', 'rest_spec_name', 'stability', 'visibility', 'behavior', 'class_serializer', 'index_privileges', 'cluster_privileges', 'doc_id'
+    'rest_spec_name', 'behavior', 'class_serializer', 'index_privileges', 'cluster_privileges', 'doc_id', 'availability'
   ]
   // in most of the cases the jsDocs comes in a single block,
   // but it can happen that the user defines multiple single line jsDoc.
@@ -602,19 +602,6 @@ export function hoistRequestAnnotations (
   setTags(jsDocs, request, tags, knownRequestAnnotations, (tags, tag, value) => {
     if (tag.endsWith('_serializer')) {
     } else if (tag === 'rest_spec_name') {
-    } else if (tag === 'visibility') {
-      if (endpoint.visibility !== null && endpoint.visibility !== undefined) {
-        assert(jsDocs, endpoint.visibility === value,
-          `Request ${request.name.name} visibility on annotation ${value} does not match spec: ${endpoint.visibility ?? ''}`)
-      }
-      endpoint.visibility = model.Visibility[value]
-    } else if (tag === 'stability') {
-      assert(jsDocs, endpoint.stability === value,
-        `Request ${request.name.name} stability on annotation ${value} does not match spec: ${endpoint.stability ?? ''}`)
-      endpoint.stability = model.Stability[value]
-    } else if (tag === 'since') {
-      assert(jsDocs, semver.valid(value), `Request ${request.name.name}'s @since is not valid semver: ${value}`)
-      endpoint.since = value
     } else if (tag === 'index_privileges') {
       const privileges = [
         'all', 'auto_configure', 'create', 'create_doc', 'create_index', 'delete', 'delete_index', 'index',
@@ -648,6 +635,33 @@ export function hoistRequestAnnotations (
       const docUrl = docIds.find(entry => entry[0] === value.trim())
       assert(jsDocs, docUrl != null, `The @doc_id '${value.trim()}' is not present in _doc_ids/table.csv`)
       endpoint.docUrl = docUrl[1]
+    } else if (tag === 'availability') {
+      // The @availability jsTag is different than most because it allows
+      // multiple values within the same docstring, hence needing to parse
+      // the values again in order to preserve multiple values.
+      const jsDocsMulti = parseJsDocTagsAllowDuplicates(jsDocs)
+      const availabilities = parseAvailabilityTags(jsDocs, jsDocsMulti.availability)
+
+      // Apply the availabilities to the Endpoint.
+      for (const [availabilityName, availabilityValue] of Object.entries(availabilities)) {
+        endpoint.availability[availabilityName] = availabilityValue
+
+        // Backfilling deprecated fields on an endpoint.
+        if (availabilityName === 'stack') {
+          if (availabilityValue.since !== undefined) {
+            endpoint.since = availabilityValue.since
+          }
+          if (availabilityValue.stability !== undefined) {
+            endpoint.stability = availabilityValue.stability
+          }
+          if (availabilityValue.visibility !== undefined) {
+            endpoint.visibility = availabilityValue.visibility
+          }
+          if (availabilityValue.featureFlag !== undefined) {
+            endpoint.featureFlag = availabilityValue.featureFlag
+          }
+        }
+      }
     } else {
       assert(jsDocs, false, `Unhandled tag: '${tag}' with value: '${value}' on request ${request.name.name}`)
     }
@@ -899,7 +913,7 @@ export function getNameSpace (node: Node): string {
 
   function cleanPath (path: string): string {
     path = dirname(path)
-      .replace(/.*[/\\]specification[/\\]?/, '')
+      .replace(/.*[/\\]specification[^/\\]*[/\\]?/, '')
       .replace(/[/\\]/g, '.')
     if (path === '') path = '_builtins'
     return path
@@ -966,6 +980,25 @@ export function parseJsDocTags (jsDoc: JSDoc[]): Record<string, string> {
   return mapped
 }
 
+/**
+ * Given a JSDoc definition, return a mapping from a tag name to all its values.
+ * This function is similar to the above parseJsDocTags() function except
+ * it allows for multiple annotations with the same name.
+ */
+export function parseJsDocTagsAllowDuplicates (jsDoc: JSDoc[]): Record<string, string[]> {
+  const mapped = {}
+  jsDoc.forEach((elem: JSDoc) => {
+    elem.getTags().forEach((tag) => {
+      const tagName = tag.getTagName()
+      if (mapped[tagName] === undefined) {
+        mapped[tagName] = []
+      }
+      mapped[tagName].push(tag.getComment() ?? '')
+    })
+  })
+  return mapped
+}
+
 /**
  * Given a JSDoc definition, it returns the Variants is present.
  * It also validates the variants syntax.
@@ -1023,6 +1056,60 @@ export function parseVariantNameTag (jsDoc: JSDoc[]): string | undefined {
   return name.replace(/'/g, '')
 }
 
+/**
+ * Parses the '@availability' JS tags into known values.
+ */
+export function parseAvailabilityTags (node: Node | Node[], values: string[]): model.Availability {
+  return values.reduce((result, value, index, array) => {
+    // Ensure that there is actually a name for this availability definition.
+    assert(node, value.split(' ').length >= 1, 'The @availability tag must include a name (either stack or serverless)')
+    const [availabilityName, ...values] = value.split(' ')
+
+    // Since we're using reduce() we need to check that there's no duplicates.
+    assert(node, !(availabilityName in result), `Duplicate @availability tag: '${availabilityName}'`)
+
+    // Enforce only known availability names.
+    assert(node, availabilityName === 'stack' || availabilityName === 'serverless', 'The @availablility <name> value must either be stack or serverless')
+
+    // Now we can parse all the key-values and load them into variables
+    // for easier access below.
+    const validKeys = ['stability', 'visibility', 'since', 'feature_flag']
+    const parsedKeyValues = parseKeyValues(node, values, ...validKeys)
+    const visibility = parsedKeyValues.visibility
+    const stability = parsedKeyValues.stability
+    const since = parsedKeyValues.since
+    const featureFlag = parsedKeyValues.feature_flag
+
+    // Remove the 'feature_flag' name used in the annotations
+    // in favor of 'featureFlag' as used in the metamodel.
+    delete parsedKeyValues.feature_flag
+
+    // Lastly we go through all the fields and validate them.
+    if (visibility !== undefined) {
+      parsedKeyValues.visibility = model.Visibility[visibility]
+      assert(node, parsedKeyValues.visibility !== undefined, `visibility is not valid: ${visibility}`)
+      if (visibility === model.Visibility.feature_flag) {
+        assert(node, featureFlag !== undefined, '\'feature_flag\' must be defined if visibility is \'feature_flag\'')
+      }
+    }
+    if (stability !== undefined) {
+      parsedKeyValues.stability = model.Stability[stability]
+      assert(node, parsedKeyValues.stability !== undefined, `stability is not valid: ${stability}`)
+    }
+    if (since !== undefined) {
+      assert(node, semver.valid(since), `'since' is not valid semver: ${since}`)
+    }
+    if (featureFlag !== undefined) {
+      assert(node, visibility === 'feature_flag', '\'visibility\' must be \'feature_flag\' if a feature flag is defined')
+      parsedKeyValues.featureFlag = featureFlag
+    }
+
+    // Add the computed set of fields to the result.
+    result[availabilityName] = parsedKeyValues
+    return result
+  }, {})
+}
+
 /**
  * Parses a list of comma-separated values as an array. Values can optionally be enclosed with single
  * or double quotes.

compiler/test/body-codegen-name/specification/_global/index/request.ts

@@ -19,8 +19,7 @@
 
 /**
  * @rest_spec_name index
- * @since 0.0.0
- * @stability stable
+ * @availability stack since=0.0.0 stability=stable
  */
 export interface Request {
   body: Foo

compiler/test/duplicate-body-codegen-name/specification/_global/index/request.ts

@@ -19,8 +19,7 @@
 
 /**
  * @rest_spec_name index
- * @since 0.0.0
- * @stability stable
+ * @availability stack since=0.0.0 stability=stable
  */
 export interface Request {
   path_parts: {

compiler/test/no-body/specification/_global/info/request.ts

@@ -19,8 +19,7 @@
 
 /**
  * @rest_spec_name info
- * @since 0.0.0
- * @stability stable
+ * @availability stack since=0.0.0 stability=stable
  */
 export interface Request {
   body: {

compiler/test/request-availability-failures/specification-duplicates/_global/index/request.ts

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @rest_spec_name index
+ * @availability stack stability=stable
+ * @availability stack stability=stable
+ */
+export interface Request<TDocument> {
+  path_parts: {
+    id?: string
+    index: string
+  }
+  query_parameters: {
+    if_primary_term?: number
+    if_seq_no?: number
+    op_type?: string
+    pipeline?: string
+    refresh?: string
+    routing?: string
+    timeout?: string
+    version?: number
+    version_type?: string
+    wait_for_active_shards?: string
+    require_alias?: boolean
+  }
+  /** @codegen_name document */
+  body?: TDocument
+}

compiler/test/request-availability-failures/specification-duplicates/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../../../specification/tsconfig.json",
+  "typeRoots": ["./**/*.ts"],
+  "include": ["./**/*.ts"]
+}

compiler/test/request-availability-failures/specification-feature-flag/_global/index/request.ts

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @rest_spec_name index
+ * @availability stack stability=stable visibility=feature_flag
+ */
+export interface Request<TDocument> {
+  path_parts: {
+    id?: string
+    index: string
+  }
+  query_parameters: {
+    if_primary_term?: number
+    if_seq_no?: number
+    op_type?: string
+    pipeline?: string
+    refresh?: string
+    routing?: string
+    timeout?: string
+    version?: number
+    version_type?: string
+    wait_for_active_shards?: string
+    require_alias?: boolean
+  }
+  /** @codegen_name document */
+  body?: TDocument
+}

compiler/test/request-availability-failures/specification-feature-flag/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../../../../specification/tsconfig.json",
+  "typeRoots": ["./**/*.ts"],
+  "include": ["./**/*.ts"]
+}

compiler/test/request-availability-failures/specification-stability/_global/index/request.ts

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * @rest_spec_name index
+ * @availability stack stability=unknown
+ */
+export interface Request<TDocument> {
+  path_parts: {
+    id?: string
+    index: string
+  }
+  query_parameters: {
+    if_primary_term?: number
+    if_seq_no?: number
+    op_type?: string
+    pipeline?: string
+    refresh?: string
+    routing?: string
+    timeout?: string
+    version?: number
+    version_type?: string
+    wait_for_active_shards?: string
+    require_alias?: boolean
+  }
+  /** @codegen_name document */
+  body?: TDocument
+}