Add support for deferred loading of bokehjs' bundles

bokehjs/src/compiler/prelude.ts

@@ -62,7 +62,7 @@ const loader = `\
     if (alias != null)
       return alias;
 
-    const trailing = name.length > 0 && name[name.lenght-1] === "/";
+    const trailing = name.length > 0 && name[name.length-1] === "/";
     const index = aliases[name + (trailing ? "" : "/") + "index"];
     if (index != null)
       return index;
@@ -104,7 +104,7 @@ const loader = `\
           });
         }
 
-        modules[id].call(mod.exports, require, mod, mod.exports, __esModule, __esExport);
+        modules[id].call(mod.exports, require, mod, mod.exports, __esModule, __esExport, base_url);
       } else {
         cache[name] = mod;
       }
@@ -195,9 +195,18 @@ ${comment(license)}
   }
   const Bokeh = root.Bokeh;
   Bokeh[bokeh.version] = bokeh;
-})(this, function() {
+})(globalThis, function() {
   let define;
-  const parent_require = typeof require === "function" && require
+  const parent_require = typeof require === "function" && require;
+  const base_url = (() => {
+    if (typeof document !== "undefined" && document.currentScript != null) {
+      const parts = document.currentScript.src.split("/");
+      parts.pop();
+      return parts.join("/");
+    } else {
+      return null
+    }
+  })()
   return ${loader}\
 `
 }
@@ -216,10 +225,10 @@ export function plugin_prelude(options?: {version?: string}): string {
 ${comment(license)}
 (function(root, factory) {
   factory(root["Bokeh"], ${str(options?.version)});
-})(this, function(Bokeh, version) {
+})(globalThis, function(Bokeh, version) {
   let define;
   return (function(modules, entry, aliases, externals) {
-    const bokeh = typeof Bokeh !== "undefined" && (version != null ? Bokeh[version] : Bokeh);
+    const bokeh = typeof Bokeh !== "undefined" ? (version != null ? Bokeh[version] : Bokeh) : null;
     if (bokeh != null) {
       return bokeh.register_plugin(modules, entry, aliases);
     } else {
@@ -233,8 +242,9 @@ export function default_prelude(options?: {global?: string}): string {
   return `\
 (function(root, factory) {
   ${options?.global != null ? `root[${str(options.global)}] = factory()` : "Object.assign(root, factory())"};
-})(this, function() {
+})(globalThis, function() {
   const parent_require = typeof require === "function" && require
+  const base_url = null
   return ${loader}
 `
 }

bokehjs/src/compiler/transforms.ts

@@ -420,7 +420,7 @@ export function wrap_in_function(module_name: string) {
   return (context: ts.TransformationContext) => (root: ts.SourceFile): ts.SourceFile => {
     const {factory} = context
     const p = (name: string) => factory.createParameterDeclaration(undefined, undefined, name)
-    const params = [p("require"), p("module"), p("exports"), p("__esModule"), p("__esExport")]
+    const params = [p("require"), p("module"), p("exports"), p("__esModule"), p("__esExport"), p("base_url")]
     const block = factory.createBlock(root.statements, true)
     const func = factory.createFunctionDeclaration(undefined, undefined, "_", undefined, params, undefined, block)
     ts.addSyntheticLeadingComment(func, ts.SyntaxKind.MultiLineCommentTrivia, ` ${module_name} `, false)

bokehjs/src/lib/client/connection.ts

@@ -162,7 +162,7 @@ export class ClientConnection {
           reject(new Error("The connection has been closed"))
         } else {
           const events: DocumentEvent[] = []
-          const document = Document.from_json(doc_json, events)
+          const document = await Document.from_json(doc_json, events)
 
           this.session = new ClientSession(this, document)
 
@@ -172,15 +172,15 @@ export class ClientConnection {
           }
 
           for (const msg of this._pending_messages) {
-            this.session.handle(msg)
+            await this.session.handle(msg)
           }
           this._pending_messages = []
 
           logger.debug("Created a new session from new pulled doc")
           resolve(this.session)
         }
       } else {
-        this.session.document.replace_with_json(doc_json)
+        await this.session.document.replace_with_json(doc_json)
         logger.debug("Updated existing session with new pulled doc")
         // Since the session already exists, we don't need to call `resolve` again.
       }

bokehjs/src/lib/client/session.ts

@@ -1,13 +1,13 @@
 import {DocumentEventBatch} from "document"
 import {ConnectionLost} from "core/bokeh_events"
-import type {Patch, Document, DocumentEvent} from "document"
+import type {InboundPatch, Document, DocumentEvent} from "document"
 import {Message} from "protocol/message"
 import type {ClientConnection} from "./connection"
 import {logger} from "core/logging"
 
 export type OkMsg = Message<{}>
 export type ErrorMsg = Message<{text: string, traceback: string | null}>
-export type PatchMsg = Message<Patch>
+export type PatchMsg = Message<InboundPatch>
 
 export class ClientSession {
   protected _document_listener = (event: DocumentEvent) => {
@@ -23,12 +23,12 @@ export class ClientSession {
     return this._connection.id
   }
 
-  handle(message: Message<unknown>): void {
+  async handle(message: Message<unknown>): Promise<void> {
     const msgtype = message.msgtype()
 
     switch (msgtype) {
       case "PATCH-DOC": {
-        this._handle_patch(message as PatchMsg)
+        await this._handle_patch(message as PatchMsg)
         break
       }
       case "OK": {
@@ -96,8 +96,8 @@ export class ClientSession {
     this._connection.send(message)
   }
 
-  protected _handle_patch(message: PatchMsg): void {
-    this.document.apply_json_patch(message.content, message.buffers)
+  protected async _handle_patch(message: PatchMsg): Promise<void> {
+    await this.document.apply_json_patch(message.content, message.buffers)
   }
 
   protected _handle_ok(message: OkMsg): void {

bokehjs/src/lib/core/util/modules.ts

@@ -15,3 +15,9 @@ export async function load_module<T>(module: Promise<T>): Promise<T | null> {
       throw e
   }
 }
+
+export async function import_url(url: string): Promise<unknown> {
+  // XXX: eval() to work around transpilation to require()
+  // https://github.com/microsoft/TypeScript/issues/43329
+  return await eval(`import("${url}")`)
+}

bokehjs/src/lib/document/document.ts

@@ -1,5 +1,6 @@
 import {default_resolver} from "../base"
 import {version as js_version} from "../version"
+import {settings} from "../core/settings"
 import {logger} from "../core/logging"
 import type {Class} from "core/class"
 import type {HasProps} from "core/has_props"
@@ -20,6 +21,7 @@ import {entries} from "core/util/object"
 import * as sets from "core/util/set"
 import type {CallbackLike} from "core/util/callbacks"
 import {execute} from "core/util/callbacks"
+import {assert} from "core/util/assert"
 import {Model} from "model"
 import type {ModelDef} from "./defs"
 import {decode_def} from "./defs"
@@ -28,6 +30,8 @@ import {DocumentReady, LODStart, LODEnd} from "core/bokeh_events"
 import type {DocumentEvent, DocumentChangedEvent, Decoded, DocumentChanged} from "./events"
 import {DocumentEventBatch, RootRemovedEvent, TitleChangedEvent, MessageSentEvent, RootAddedEvent} from "./events"
 
+declare const base_url: string | null | undefined
+
 Deserializer.register("model", decode_def)
 
 export type Out<T> = T
@@ -62,6 +66,7 @@ export class EventManager {
 export type DocJson = {
   version?: string
   title?: string
+  bundles?: string[]
   defs?: ModelDef[]
   roots: ModelRep[]
   callbacks?: {[key: string]: ModelRep[]}
@@ -71,6 +76,10 @@ export type Patch = {
   events: DocumentChanged[]
 }
 
+export type InboundPatch = Patch & {
+  bundles?: string[]
+}
+
 export const documents: Document[] = []
 
 export const DEFAULT_TITLE = "Bokeh Application"
@@ -427,9 +436,9 @@ export class Document implements Equatable {
     }
   }
 
-  static from_json_string(s: string, events?: Out<DocumentEvent[]>): Document {
+  static async from_json_string(s: string, events?: Out<DocumentEvent[]>): Promise<Document> {
     const json = JSON.parse(s)
-    return Document.from_json(json, events)
+    return await Document.from_json(json, events)
   }
 
   private static _handle_version(json: DocJson): void {
@@ -448,10 +457,65 @@ export class Document implements Equatable {
     }
   }
 
-  static from_json(doc_json: DocJson, events?: Out<DocumentEvent[]>): Document {
+  private static _known_bundles: Set<string> = new Set()
+
+  static async load_bundles(bundles: string[]): Promise<void> {
+
+    const is_absolute = (uri: string): boolean => {
+      return uri.startsWith("http://") || uri.startsWith("https://") || uri.startsWith("/")
+    }
+
+    const load_url = async (url: string): Promise<boolean> => {
+      if (this._known_bundles.has(url)) {
+        return true
+      }
+
+      const response = await fetch(url)
+      if (response.ok) {
+        this._known_bundles.add(url)
+        const module = await response.text()
+        eval(module)
+        return true
+      } else {
+        return false
+      }
+    }
+
+    const build_url = (base_name: string, version?: string): string => {
+      assert(base_url != null)
+      const name = version != null ? `${base_name}-${version}` : base_name
+      const suffix = settings.dev ? ".min.js" : ".js"
+      return `${base_url}/${name}${suffix}`
+    }
+
+    for (const bundle of bundles) {
+      if (is_absolute(bundle)) {
+        if (await load_url(bundle)) {
+          continue
+        }
+      } else {
+        const url = build_url(bundle)
+        if (await load_url(url)) {
+          continue
+        }
+        const versioned_url = build_url(bundle, pyify_version(js_version))
+        if (await load_url(versioned_url)) {
+          continue
+        }
+      }
+
+      console.error(`failed to load '${bundle}' bundle`)
+    }
+  }
+
+  static async from_json(doc_json: DocJson, events?: Out<DocumentEvent[]>): Promise<Document> {
     logger.debug("Creating Document from JSON")
     Document._handle_version(doc_json)
 
+    if (doc_json.bundles != null) {
+      await this.load_bundles(doc_json.bundles)
+    }
+
     const resolver = new ModelResolver(default_resolver)
     if (doc_json.defs != null) {
       const deserializer = new Deserializer(resolver)
@@ -494,8 +558,8 @@ export class Document implements Equatable {
     return doc
   }
 
-  replace_with_json(json: DocJson): void {
-    const replacement = Document.from_json(json)
+  async replace_with_json(json: DocJson): Promise<void> {
+    const replacement = await Document.from_json(json)
     replacement.destructively_move(this)
   }
 
@@ -520,7 +584,11 @@ export class Document implements Equatable {
     return patch
   }
 
-  apply_json_patch(patch: Patch, buffers: Map<ID, ArrayBuffer> = new Map()): void {
+  async apply_json_patch(patch: InboundPatch, buffers: Map<ID, ArrayBuffer> = new Map()): Promise<void> {
+    if (patch.bundles != null) {
+      await Document.load_bundles(patch.bundles)
+    }
+
     this._push_all_models_freeze()
 
     const deserializer = new Deserializer(this._resolver, this._all_models, (obj) => obj.attach_document(this))

bokehjs/src/lib/embed/index.ts

@@ -56,7 +56,7 @@ async function _embed_items(docs_json: string | DocsJson, render_items: RenderIt
 
   const docs: {[key: string]: Document} = {}
   for (const [docid, doc_json] of entries(docs_json)) {
-    docs[docid] = Document.from_json(doc_json)
+    docs[docid] = await Document.from_json(doc_json)
   }
 
   const views: ViewManager[] = []

bokehjs/src/lib/embed/notebook.ts

@@ -1,4 +1,4 @@
-import type {Patch} from "document"
+import type {InboundPatch} from "document"
 import {Document} from "document"
 import {Receiver} from "protocol/receiver"
 import type {Message} from "protocol/message"
@@ -14,7 +14,7 @@ import {_resolve_element, _resolve_root_elements} from "./dom"
 // This has to be available at Bokeh.embed.kernels in JupyterLab.
 export const kernels: {[key: string]: unknown} = {}
 
-function _handle_notebook_comms(this: Document, receiver: Receiver, comm_msg: CommMessage): void {
+async function _handle_notebook_comms(this: Document, receiver: Receiver, comm_msg: CommMessage): Promise<void> {
   if (comm_msg.buffers.length > 0) {
     receiver.consume(comm_msg.buffers[0].buffer)
   } else {
@@ -23,7 +23,7 @@ function _handle_notebook_comms(this: Document, receiver: Receiver, comm_msg: Co
 
   const msg = receiver.message
   if (msg != null) {
-    this.apply_json_patch((msg as Message<Patch>).content, msg.buffers)
+    await this.apply_json_patch((msg as Message<InboundPatch>).content, msg.buffers)
   }
 }
 
@@ -52,7 +52,7 @@ function _init_comms(target: string, doc: Document): void {
     } catch (e) {
       logger.warn(`Jupyter comms failed to register. push_notebook() will not function. (exception reported: ${e})`)
     }
-  } else if  (typeof google != "undefined" && google.colab.kernel != null) {
+  } else if (typeof google != "undefined" && google.colab.kernel != null) {
     logger.info(`Registering Google Colab comms for target ${target}`)
     const comm_manager = google.colab.kernel.comms
     try {
@@ -66,7 +66,7 @@ function _init_comms(target: string, doc: Document): void {
             buffers.push(new DataView(buffer))
           }
           const msg = {content, buffers}
-          _handle_notebook_comms.bind(doc)(r, msg)
+          await _handle_notebook_comms.bind(doc)(r, msg)
         }
       })
     } catch (e) {
@@ -82,7 +82,7 @@ export async function embed_items_notebook(docs_json: DocsJson, render_items: Re
     throw new Error("embed_items_notebook expects exactly one document in docs_json")
   }
 
-  const document = Document.from_json(values(docs_json)[0])
+  const document = await Document.from_json(values(docs_json)[0])
 
   for (const item of render_items) {
     if (item.notebook_comms_target != null) {

bokehjs/src/lib/models/callbacks/customjs.ts

@@ -6,7 +6,8 @@ import {use_strict} from "core/util/string"
 
 import type {Model} from "../../model"
 import {logger} from "core/logging"
-import {isFunction} from "core/util/types"
+import {isPlainObject, isFunction} from "core/util/types"
+import {import_url} from "core/util/modules"
 import type {ViewManager} from "core/view"
 import {index} from "embed/standalone"
 
@@ -57,10 +58,8 @@ export class CustomJS extends Callback {
   protected async _compile_module(): Promise<ESFunc> {
     const url = URL.createObjectURL(new Blob([this.code], {type: "text/javascript"}))
     try {
-      // XXX: eval() to work around transpilation to require()
-      // https://github.com/microsoft/TypeScript/issues/43329
-      const module = await eval(`import("${url}")`)
-      if (isFunction(module.default)) {
+      const module = await import_url(url)
+      if (isPlainObject(module) && isFunction(module.default)) {
         return module.default as ESFunc
       } else {
         logger.warn("custom ES module didn't export a default function")

bokehjs/test/integration/cross.ts

@@ -9,7 +9,7 @@ async function test(name: string) {
   const response = await fetch(`/cases/${name}`)
   const text = await response.text()
   const doc_json = json5.parse<DocJson>(text)
-  const doc = Document.from_json(doc_json)
+  const doc = await Document.from_json(doc_json)
   return await display(doc)
 }