Split intrinsic from std lib (microsoft#3170)

fix microsoft#3171
Currently the `--nostdlib` option is just useless, here splitting the
intrinsic types from the rest of the std lib.

Intrinsic types will always be loaded regardless of what `nostdlib`
says.


Being able to disable that can be useful in the compiler test to not be
polluted by some extra elements.

Author: timotheeguerin

.chronus/changes/actual-std-lib-2024-3-15-15-53-0.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+`--nostdlib` flag will now work by only applying to optional standard library types

.chronus/changes/actual-std-lib-2024-3-15-15-55-33.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/playground"
+---
+
+Add support for new intrinsic vs std lib split in the compiler. 

packages/compiler/lib/lib.tsp packages/compiler/lib/intrinsics.tsp

@@ -1,3 +1,6 @@
+import "../dist/src/lib/intrinsic-decorators.js";
+
+// This file contains all the intrinsic types of typespec. Everything here will always be loaded
 namespace TypeSpec;
 
 /**
@@ -122,20 +125,6 @@ scalar duration;
  */
 scalar boolean;
 
-/**
- * Represent a 32-bit unix timestamp datetime with 1s of granularity.
- * It measures time by the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970.
- */
-@encode("unixTimestamp", int32)
-scalar unixTimestamp32 extends utcDateTime;
-
-/**
- * Represent a model
- */
-// Deprecated June 2023 sprint
-#deprecated "object is deprecated. Please use {} for an empty model, `Record<unknown>` for a record with unknown property types, `unknown[]` for an array."
-model object {}
-
 /**
  * @dev Array model type, equivalent to `Element[]`
  * @template Element The type of the array elements
@@ -149,64 +138,3 @@ model Array<Element> {}
  */
 @indexer(string, Element)
 model Record<Element> {}
-
-/**
- * Represent a URL string as described by https://url.spec.whatwg.org/
- */
-scalar url extends string;
-
-/**
- * Represents a collection of optional properties.
- *
- * @template Source An object whose spread properties are all optional.
- */
-@doc("The template for adding optional properties.")
-@withOptionalProperties
-model OptionalProperties<Source> {
-  ...Source;
-}
-
-/**
- * Represents a collection of updateable properties.
- *
- * @template Source An object whose spread properties are all updateable.
- */
-@doc("The template for adding updateable properties.")
-@withUpdateableProperties
-model UpdateableProperties<Source> {
-  ...Source;
-}
-
-/**
- * Represents a collection of omitted properties.
- *
- * @template Source An object whose properties are spread.
- * @template Keys The property keys to omit.
- */
-@doc("The template for omitting properties.")
-@withoutOmittedProperties(Keys)
-model OmitProperties<Source, Keys extends string> {
-  ...Source;
-}
-
-/**
- * Represents a collection of properties with default values omitted.
- *
- * @template Source An object whose spread property defaults are all omitted.
- */
-@withoutDefaultValues
-model OmitDefaults<Source> {
-  ...Source;
-}
-
-/**
- * Applies a visibility setting to a collection of properties.
- *
- * @template Source An object whose properties are spread.
- * @template Visibility The visibility to apply to all properties.
- */
-@doc("The template for setting the default visibility of key properties.")
-@withDefaultKeyVisibility(Visibility)
-model DefaultKeyVisibility<Source, Visibility extends valueof string> {
-  ...Source;
-}

packages/compiler/lib/main.tsp

@@ -1,4 +1,4 @@
-import "../dist/src/lib/decorators.js";
+import "../../dist/src/lib/decorators.js";
 
 using TypeSpec.Reflection;
 

packages/compiler/lib/decorators.tsp packages/compiler/lib/std/decorators.tsp

@@ -0,0 +1,5 @@
+// TypeSpec standard library. Everything in here can be omitted by using `--nostdlib` cli flag or `nostdlib` in the config.
+import "./types.tsp";
+import "./decorators.tsp";
+import "./reflection.tsp";
+import "./projected-names.tsp";

packages/compiler/lib/std/main.tsp

@@ -0,0 +1,76 @@
+namespace TypeSpec;
+
+/**
+ * Represent a 32-bit unix timestamp datetime with 1s of granularity.
+ * It measures time by the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970.
+ */
+@encode("unixTimestamp", int32)
+scalar unixTimestamp32 extends utcDateTime;
+
+/**
+ * Represent a model
+ */
+// Deprecated June 2023 sprint
+#deprecated "object is deprecated. Please use {} for an empty model, `Record<unknown>` for a record with unknown property types, `unknown[]` for an array."
+model object {}
+
+/**
+ * Represent a URL string as described by https://url.spec.whatwg.org/
+ */
+scalar url extends string;
+
+/**
+ * Represents a collection of optional properties.
+ *
+ * @template Source An object whose spread properties are all optional.
+ */
+@doc("The template for adding optional properties.")
+@withOptionalProperties
+model OptionalProperties<Source> {
+  ...Source;
+}
+
+/**
+ * Represents a collection of updateable properties.
+ *
+ * @template Source An object whose spread properties are all updateable.
+ */
+@doc("The template for adding updateable properties.")
+@withUpdateableProperties
+model UpdateableProperties<Source> {
+  ...Source;
+}
+
+/**
+ * Represents a collection of omitted properties.
+ *
+ * @template Source An object whose properties are spread.
+ * @template Keys The property keys to omit.
+ */
+@doc("The template for omitting properties.")
+@withoutOmittedProperties(Keys)
+model OmitProperties<Source, Keys extends string> {
+  ...Source;
+}
+
+/**
+ * Represents a collection of properties with default values omitted.
+ *
+ * @template Source An object whose spread property defaults are all omitted.
+ */
+@withoutDefaultValues
+model OmitDefaults<Source> {
+  ...Source;
+}
+
+/**
+ * Applies a visibility setting to a collection of properties.
+ *
+ * @template Source An object whose properties are spread.
+ * @template Visibility The visibility to apply to all properties.
+ */
+@doc("The template for setting the default visibility of key properties.")
+@withDefaultKeyVisibility(Visibility)
+model DefaultKeyVisibility<Source, Visibility extends valueof string> {
+  ...Source;
+}

packages/compiler/lib/projected-names.tsp packages/compiler/lib/std/projected-names.tsp

@@ -19,7 +19,7 @@
   ],
   "type": "module",
   "main": "dist/src/index.js",
-  "tspMain": "lib/main.tsp",
+  "tspMain": "lib/std/main.tsp",
   "exports": {
     ".": {
       "types": "./dist/src/index.d.ts",
@@ -54,7 +54,7 @@
     "tsp-server": "cmd/tsp-server.js"
   },
   "files": [
-    "lib/*.tsp",
+    "lib/**/*.tsp",
     "dist/**",
     "templates/**",
     "entrypoints",

packages/compiler/lib/reflection.tsp packages/compiler/lib/std/reflection.tsp

@@ -1,4 +1,5 @@
-import { $docFromComment, getIndexer, isArrayModelType } from "../lib/decorators.js";
+import { $docFromComment, isArrayModelType } from "../lib/decorators.js";
+import { getIndexer } from "../lib/intrinsic-decorators.js";
 import { MultiKeyMap, Mutable, createRekeyableMap, isArray, mutate } from "../utils/misc.js";
 import { createSymbol, createSymbolTable } from "./binder.js";
 import { createChangeIdentifierCodeFix } from "./compiler-code-fixes/change-identifier.codefix.js";
@@ -356,9 +357,6 @@ export function createChecker(program: Program): Checker {
 
   const typespecNamespaceBinding = globalNamespaceNode.symbol.exports!.get("TypeSpec");
   if (typespecNamespaceBinding) {
-    // the TypeSpec namespace binding will be absent if we've passed
-    // the no-std-lib option.
-    // the first declaration here is the JS file for the TypeSpec script.
     initializeTypeSpecIntrinsics();
     for (const file of program.sourceFiles.values()) {
       addUsingSymbols(typespecNamespaceBinding.exports!, file.locals);

packages/compiler/lib/std/types.tsp

@@ -28,7 +28,7 @@ export const NodeHost: CompilerHost = {
   getJsImport: (path: string) => import(pathToFileURL(path).href),
   getLibDirs() {
     const rootDir = this.getExecutionRoot();
-    return [joinPaths(rootDir, "lib")];
+    return [joinPaths(rootDir, "lib/std")];
   },
   stat(path: string) {
     return stat(path);

packages/compiler/package.json

@@ -34,7 +34,7 @@ import {
 } from "./module-resolver.js";
 import { CompilerOptions } from "./options.js";
 import { isImportStatement, parse, parseStandaloneTypeReference } from "./parser.js";
-import { getDirectoryPath, joinPaths } from "./path-utils.js";
+import { getDirectoryPath, joinPaths, resolvePath } from "./path-utils.js";
 import { createProjector } from "./projector.js";
 import { createSourceFile } from "./source-file.js";
 import {
@@ -332,8 +332,9 @@ export async function compile(
   }
   const binder = createBinder(program);
 
+  await loadIntrinsicTypes();
   if (!options?.nostdlib) {
-    await loadStandardLibrary(program);
+    await loadStandardLibrary();
   }
 
   // Load additional imports prior to compilation
@@ -458,7 +459,16 @@ export async function compile(
     }
   }
 
-  async function loadStandardLibrary(program: Program) {
+  async function loadIntrinsicTypes() {
+    const locationContext: LocationContext = { type: "compiler" };
+    await loadTypeSpecFile(
+      resolvePath(host.getExecutionRoot(), "lib/intrinsics.tsp"),
+      locationContext,
+      NoTarget
+    );
+  }
+
+  async function loadStandardLibrary() {
     const locationContext: LocationContext = { type: "compiler" };
     for (const dir of host.getLibDirs()) {
       await loadDirectory(dir, locationContext, NoTarget);

packages/compiler/src/core/checker.ts

@@ -59,7 +59,6 @@ import {
   EnumMember,
   Interface,
   Model,
-  ModelIndexer,
   ModelProperty,
   Namespace,
   Operation,
@@ -287,16 +286,6 @@ export const $inspectTypeName: InspectTypeNameDecorator = (context, target: Type
   console.log(getTypeName(target));
 };
 
-const indexTypeKey = createStateSymbol("index");
-export const $indexer = (context: DecoratorContext, target: Type, key: Scalar, value: Type) => {
-  const indexer: ModelIndexer = { key, value };
-  context.program.stateMap(indexTypeKey).set(target, indexer);
-};
-
-export function getIndexer(program: Program, target: Type): ModelIndexer | undefined {
-  return program.stateMap(indexTypeKey).get(target);
-}
-
 export function isStringType(program: Program | ProjectedProgram, target: Type): target is Scalar {
   const coreType = program.checker.getStdType("string");
   const stringType = target.projector ? target.projector.projectType(coreType) : coreType;

packages/compiler/src/core/node-host.ts

@@ -0,0 +1,14 @@
+import { Program } from "../core/program.js";
+import { DecoratorContext, ModelIndexer, Scalar, Type } from "../core/types.js";
+
+export const namespace = "TypeSpec";
+
+const indexTypeKey = Symbol.for(`TypeSpec.index`);
+export const $indexer = (context: DecoratorContext, target: Type, key: Scalar, value: Type) => {
+  const indexer: ModelIndexer = { key, value };
+  context.program.stateMap(indexTypeKey).set(target, indexer);
+};
+
+export function getIndexer(program: Program, target: Type): ModelIndexer | undefined {
+  return program.stateMap(indexTypeKey).get(target);
+}

packages/compiler/src/core/program.ts

@@ -42,7 +42,7 @@ function createTestCompilerHost(
   jsImports: Map<string, Record<string, any>>,
   options?: TestHostOptions
 ): CompilerHost {
-  const libDirs = [resolveVirtualPath(".tsp/lib")];
+  const libDirs = [resolveVirtualPath(".tsp/lib/std")];
   if (!options?.excludeTestLib) {
     libDirs.push(resolveVirtualPath(".tsp/test-lib"));
   }
@@ -224,7 +224,7 @@ export const StandardTestLibrary: TypeSpecTestLibrary = {
   packageRoot: await findTestPackageRoot(import.meta.url),
   files: [
     { virtualPath: "./.tsp/dist/src/lib", realDir: "./dist/src/lib", pattern: "*" },
-    { virtualPath: "./.tsp/lib", realDir: "./lib", pattern: "*" },
+    { virtualPath: "./.tsp/lib", realDir: "./lib", pattern: "**" },
   ],
 };
 

packages/compiler/src/lib/decorators.ts

@@ -127,7 +127,15 @@ describe("compiler: semantic walker", () => {
 
     deepStrictEqual(
       result.namespaces.map((x) => getNamespaceFullName(x)),
-      ["", "Global", "Global.My", "Global.My.Simple", "Global.My.Parent", "Global.My.Parent.Child"]
+      [
+        "",
+        "TypeSpec",
+        "Global",
+        "Global.My",
+        "Global.My.Simple",
+        "Global.My.Parent",
+        "Global.My.Parent.Child",
+      ]
     );
   });