Add logic to create @webref/css package (#115)

Code follows the same structure as for the `@webref/idl` package. Tests
make sure that all CSS fragments can be parsed with the Value Definition Syntax
parser of CSSTree.

A couple of CSS patches need to be applied to the extracts to fix value definitions.

Tests currently only guarantee that the value definitions can be parsed with CSSTree,
with the exception of a handful value definitions that are a priori valid but that
cannot be parsed with the CSSTree parser for the time being, see discussion in:
w3c/reffy#494 (comment)

Co-authored-by: Philip Jägenstedt <philip@foolip.org>

.gitignore

@@ -1,3 +1,5 @@
 node_modules/
+packages/css/*.json
+!packages/css/package.json
 packages/idl/*.idl
 wpt/

ed/csspatches/README.md

@@ -0,0 +1,5 @@
+# CSS patches
+
+These are patches applied to the CSS extracts scraped from specs to produce the `@webref/css` package. These patches can break as specs are updated and thus need ongoing maintenance.
+
+For details on how to create and update patches, please see the [Web IDL patches documentation](../idlpatches/README.md)

ed/csspatches/SVG.json.patch

@@ -0,0 +1,34 @@
+From ca6c95e3f6482926c9b919e581ba340d096d020c Mon Sep 17 00:00:00 2001
+From: Francois Daoust <fd@tidoust.net>
+Date: Thu, 4 Mar 2021 15:41:15 +0100
+Subject: [PATCH] Replace curly quotes with simple quotes
+
+---
+ ed/css/SVG.json | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/ed/css/SVG.json b/ed/css/SVG.json
+index 4bf93e4b1..8c47bb377 100644
+--- a/ed/css/SVG.json
++++ b/ed/css/SVG.json
+@@ -182,7 +182,7 @@
+     },
+     "fill-opacity": {
+       "name": "fill-opacity",
+-      "value": "<‘opacity’>",
++      "value": "<'opacity'>",
+       "initial": "1",
+       "appliesTo": "shapes and text content elements",
+       "inherited": "yes",
+@@ -204,7 +204,7 @@
+     },
+     "stroke-opacity": {
+       "name": "stroke-opacity",
+-      "value": "<‘opacity’>",
++      "value": "<'opacity'>",
+       "initial": "1",
+       "appliesTo": "shapes and text content elements",
+       "inherited": "yes",
+-- 
+2.30.1.windows.1
+

ed/csspatches/css-page.json.patch

@@ -0,0 +1,25 @@
+From b03e6c9108b37d5888980765afb2844b8f7cec99 Mon Sep 17 00:00:00 2001
+From: Francois Daoust <fd@tidoust.net>
+Date: Thu, 4 Mar 2021 16:09:10 +0100
+Subject: [PATCH] Drop invalid comment in value definition
+
+---
+ ed/css/css-page.json | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/ed/css/css-page.json b/ed/css/css-page.json
+index bfcaef74d..759e0cae0 100644
+--- a/ed/css/css-page.json
++++ b/ed/css/css-page.json
+@@ -54,7 +54,7 @@
+       "value": "[ <ident-token>? <pseudo-page>* ]!"
+     },
+     "<pseudo-page>": {
+-      "value": "':' [ left | right | first | blank ] /* Margin rules */"
++      "value": "':' [ left | right | first | blank ]"
+     },
+     "<page-size>": {
+       "prose": "A page size can be specified using one of the following media names. This is the equivalent of specifying size using length values. The definition of the media names comes from Media Standardized Names [PWGMSN]. A5 Equivalent to the size of ISO A5 media: 148mm wide and 210 mm high. A4 Equivalent to the size of ISO A4 media: 210 mm wide and 297 mm high. A3 Equivalent to the size of ISO A3 media: 297mm wide and 420mm high. B5 Equivalent to the size of ISO B5 media: 176mm wide by 250mm high. B4 Equivalent to the size of ISO B4 media: 250mm wide by 353mm high. JIS-B5 Equivalent to the size of JIS B5 media: 182mm wide by 257mm high. JIS-B4 Equivalent to the size of JIS B4 media: 257mm wide by 364mm high. letter Equivalent to the size of North American letter media: 8.5 inches wide and 11 inches high legal Equivalent to the size of North American legal: 8.5 inches wide by 14 inches high. ledger Equivalent to the size of North American ledger: 11 inches wide by 17 inches high."
+-- 
+2.30.1.windows.1
+

package.json

@@ -25,7 +25,9 @@
   },
   "devDependencies": {
     "@octokit/rest": "16.28.7",
+    "@webref/css": "file:packages/css",
     "@webref/idl": "file:packages/idl",
+    "css-tree": "1.1.2",
     "flags": "0.1.3",
     "lerna": "3.22.1",
     "mocha": "8.2.0",

packages/css/README.md

@@ -0,0 +1,43 @@
+# CSS definitions of the web platform
+
+This package contains CSS property definitions scraped from the latest versions of web platform specifications in [webref](https://github.com/w3c/webref), with fixes applied to ensure ([almost](#guarantees)) all CSS value definitions can be parsed with [CSSTree](https://github.com/csstree/csstree).
+
+# API
+
+The async `listAll()` method resolves with an object where the keys are spec shortnames, and the values are the data for that spec. Example:
+
+```js
+const css = require('@webref/css');
+
+const parsedFiles = await css.listAll();
+for (const [shortname, data] of Object.entries(parsedFiles)) {
+  // do something with the json object
+}
+```
+
+CSS fragments that appear in the objects, in other words the contents of the `properties[].value`, `properties[].newValues`, `descriptors[].value` and `valuespaces[].value` properties can be parsed with the [CSSTree Value Definition Syntax parser](https://github.com/csstree/csstree/blob/master/docs/definition-syntax.md#value-definition-syntax). Example:
+
+```js
+const css = require('@webref/css');
+const { definitionSyntax } = require('css-tree');
+
+const parsedFiles = await css.listAll();
+for (const [shortname, data] of Object.entries(parsedFiles)) {
+  for (const [name, desc] of Object.entries(data.properties)) {
+    if (desc.value) {
+      try {
+        const ast = definitionSyntax.parse(desc.value);
+        // do something with the ast
+      }
+      catch {
+        // one of the few value definitions that cannot yet be parsed by CSSTree
+      }
+    }
+  }
+}
+```
+
+# Guarantees
+
+The following guarantees are provided by this package:
+- All CSS files can be parsed by the version of [CSSTree](https://github.com/csstree/csstree) used in `peerDependencies` in `package.json`, with the exception of a handful CSS value definitions that, although valid, are not yet supported by CSSTree.

packages/css/index.js

@@ -0,0 +1,16 @@
+const fs = require('fs').promises;
+const path = require('path');
+
+async function listAll() {
+  const all = {};
+  const files = await fs.readdir(__dirname);
+  for (const f of files) {
+    if (f.endsWith('.json') && f !== 'package.json') {
+      const text = await fs.readFile(path.join(__dirname, f), 'utf8');
+      all[path.basename(f, '.json')] = JSON.parse(text);
+    }
+  }
+  return all;
+}
+
+module.exports = {listAll};

packages/css/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@webref/css",
+  "description": "CSS definitions of the web platform",
+  "version": "1.0.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/w3c/webref.git"
+  },
+  "bugs": {
+    "url": "https://github.com/w3c/webref/issues"
+  },
+  "license": "MIT",
+  "main": "index.js",
+  "peerDependencies": {
+    "css-tree": "^1.1.2"
+  }
+}

packages/prepare.js

@@ -4,38 +4,55 @@ const util = require('util');
 const execFile = util.promisify(require('child_process').execFile);
 
 const rootDir = path.resolve(__dirname, '..');
-const srcDir = path.join(rootDir, 'ed/idl');
-const patchDir = path.join(rootDir, 'ed/idlpatches');
-const dstDir = path.join(rootDir, 'packages/idl');
+
+const packages = [
+  {
+    name: 'idl',
+    srcDir: path.join(rootDir, 'ed/idl'),
+    dstDir: path.join(rootDir, 'packages/idl'),
+    patchDir: path.join(rootDir, 'ed/idlpatches'),
+    fileExt: 'idl'
+  },
+  {
+    name: 'css',
+    srcDir: path.join(rootDir, 'ed/css'),
+    dstDir: path.join(rootDir, 'packages/css'),
+    patchDir: path.join(rootDir, 'ed/csspatches'),
+    fileExt: 'json'
+  },
+];
+
 
 async function main() {
-  // rm dstDir/*.idl
-  const dstFiles = await fs.readdir(dstDir);
-  for (const file of dstFiles) {
-    if (file.endsWith('.idl')) {
-      await fs.unlink(path.join(dstDir, file));
+  for (const { name, srcDir, dstDir, patchDir, fileExt } of packages) {
+    // rm dstDir/*.${fileExt}
+    const dstFiles = await fs.readdir(dstDir);
+    for (const file of dstFiles) {
+      if (file.endsWith(`.${fileExt}`) && file !== 'package.json') {
+        await fs.unlink(path.join(dstDir, file));
+      }
     }
-  }
 
-  // cp srcDir/*.idl dstDir/
-  const srcFiles = await fs.readdir(srcDir);
-  for (const file of srcFiles) {
-    if (file.endsWith('.idl')) {
-      await fs.copyFile(path.join(srcDir, file), path.join(dstDir, file));
+    // cp srcDir/*.${fileExt} dstDir/
+    const srcFiles = await fs.readdir(srcDir);
+    for (const file of srcFiles) {
+      if (file.endsWith('.' + fileExt)) {
+        await fs.copyFile(path.join(srcDir, file), path.join(dstDir, file));
+      }
     }
-  }
 
-  // The patches are against ed/idl/ and can be applied there using `git am`,
-  // but apply them in packages/idl/ instead using `git apply`.
-  // See ed/idlpatches/README.md for how to maintain these patches.
-  const patchFiles = await fs.readdir(patchDir);
-  for (const file of patchFiles) {
-    if (file.endsWith('.patch')) {
-      const patch = path.join(patchDir, file);
-      console.log(`Applying ${file}`);
-      await execFile('git', ['apply', '--directory=packages/idl', '-p3', patch], {
-        cwd: rootDir
-      });
+    // The patches are against srcDir and can be applied there using `git am`,
+    // but apply them in dstDir instead using `git apply`.
+    // See ed/idlpatches/README.md for how to maintain these patches.
+    const patchFiles = await fs.readdir(patchDir);
+    for (const file of patchFiles) {
+      if (file.endsWith('.patch')) {
+        const patch = path.join(patchDir, file);
+        console.log(`Applying ${patch}`);
+        await execFile('git', ['apply', `--directory=packages/${name}`, '-p3', patch], {
+          cwd: rootDir
+        });
+      }
     }
   }
 }

test/css/all.js

@@ -0,0 +1,15 @@
+const assert = require('assert').strict;
+
+const css = require('@webref/css');
+
+describe('@webidl/css module', () => {
+  it('listAll', async () => {
+    const all = await css.listAll();
+    assert(Object.keys(all).length > 0);
+    for (const desc of Object.values(all)) {
+      assert(desc);
+      assert(desc.spec);
+      assert(desc.spec.title);
+    }
+  });
+});

test/css/package.js

@@ -0,0 +1,11 @@
+const assert = require('assert').strict;
+
+const cssPackage = require('../../packages/css/package.json');
+const rootPackage = require('../../package.json');
+
+describe('@webidl/css package', () => {
+  it('uses same version as main package', () => {
+    assert.equal(cssPackage.peerDependencies['css-tree'],
+                 `^${rootPackage.devDependencies['css-tree']}`);
+  });
+});

test/css/parse.js

@@ -0,0 +1,45 @@
+const assert = require('assert').strict;
+const { definitionSyntax } = require('css-tree');
+
+const css = require('@webref/css');
+
+const cssValues = [
+  { type: 'property', prop: 'properties', value: 'value' },
+  { type: 'extended property', prop: 'properties', value: 'newValues' },
+  { type: 'descriptor', prop: 'descriptors', value: 'value' },
+  { type: 'value space', prop: 'valuespaces', value: 'value' }
+];
+
+// TEMP: constructs that are not yet supported by the parser (2021-03-10)
+// See: https://github.com/w3c/reffy/issues/494#issuecomment-790713119
+const tempIgnore = [
+  { shortname: 'css-extensions', prop: 'valuespaces', name: '<custom-selector>' },
+  { shortname: 'fill-stroke', prop: 'properties', name: 'stroke-dasharray' },
+  { shortname: 'svg-animations', prop: 'valuespaces', name: '<control-point>' },
+  { shortname: 'svg-markers', prop: 'properties', name: 'marker' },
+  { shortname: 'svg-strokes', prop: 'valuespaces', name: '<dasharray>' }
+];
+
+css.listAll().then(all => {
+  for (const [shortname, data] of Object.entries(all)) {
+    describe(`The ${shortname} entry in @webidl/css`, () => {
+      for (const { type, prop, value } of cssValues) {
+        for (const [name, desc] of Object.entries(data[prop])) {
+          if (!desc[value]) {
+            continue;
+          }
+          if (tempIgnore.some(c => c.shortname === shortname &&
+              c.prop === prop && c.name === name)) {
+            continue;
+          }
+          it(`defines a valid ${type} "${name}"`, () => {
+            assert.doesNotThrow(() => {
+              const ast = definitionSyntax.parse(desc[value]);
+              assert(ast.type);
+            }, `Invalid definition value: ${desc[value]}`);
+          });
+        }
+      }
+    });
+  }
+}).then(run);