Add scripts for generating sample list (#963)

* init project

* init .gitignore

* Add main scripts

* Add prefetch script

* Add extension-apis.json

* Add README.md

* Update comments

* Add newline to .gitignore

* Update type definition

* Update extension api json loading.

* Update README.md

* Update deps

* Pin deps versions

* Update extension-apis.json

* Update prefetch script

* Update script

* Remove type definitions prefix

* Add comment to auto generated json

* Split utils

* Update types

* Add manifest utils

* refactor

* Reject if there is a babel error.

* Add parallel support

* Remove redundant import

* loadExtensionApis execute synchronously

* Add test deps

* Fix unexpected result

* Add test scripts

* Update type definition

* Add start function

* Rename api variable

* Fix test

* Update getApiType method

* Remove singularize

* Fix loadExtensionApis return signature

* Remove parallel controller

* Update type definition

* Add comments

* Update types

* Update api detector

* Fix api test

* Add more test cases

* Remove `$special` key

* Fix typo

* Simplify code

* Rename `apiType` to `type`

* Add warning if no api found

* Rename variable

* Update getFullMemberExpression

* Remove the special case for `chrome.storage`.

* Add getManifest function

* Revoke changes

* Fix crash caused by incorrect folder detection

* Remove redundant import

* Check property type with typedoc

* Add action

* Fix wrong node version

* Update code

* Rename variable

* Remove unnecessary Map

* Update README

* Fix typo

* Remove unnecessary output

* Add comments

* Remove the judgment on storage API.

* Update test case

* Update comments

* Extract `getAllJsFile()` into the filesystem util

* Explain how to run the tests in README

* Pin deps

* Extract variable

* Extract `isDirectory()` into the filesystem util

* Remove assertion

* Update the parameter of `extractApiCalls()`

* Update tests

* Remove action
This commit is contained in:
Xuezhou Dai
2023-08-01 17:00:30 +08:00
committed by GitHub
parent 6f2e616494
commit 9d9272809b
15 changed files with 6461 additions and 0 deletions

View File

@@ -0,0 +1 @@
extension-samples.json

View File

@@ -0,0 +1,153 @@
# Sample List Generator
## Overview
It's a script that generates `./extension-samples.json` with the list of all the samples available. Currently, this JSON will be provided to [developer.chrome.com](https://developer.chrome.com) for generating a list page containing all the samples. This allows developers to quickly find the sample they want to reference.
## How to use
### Install dependencies
```bash
npm install
```
### Run prefetch script (optional)
The prefetch script will generate a list of all the available extension apis on [developer.chrome.com](https://developer.chrome.com/docs/extensions/reference) and save it to `./extension-apis.json`.
The file `./extension-apis.json` will be committed so you don't need to run this script unless you want to update the list.
```bash
npm run prepare-chrome-types
```
### Run the generator
```bash
npm start
```
### Run the tests
```bash
npm test
```
## Types
```ts
type ApiTypeResult = 'event' | 'method' | 'property' | 'type' | 'unknown';
interface ApiItem {
type: ApiTypeResult;
namespace: string;
propertyName: string;
}
interface SampleItem {
type: 'API_SAMPLE' | 'FUNCTIONAL_SAMPLE';
name: string;
title: string;
description: string;
repo_link: string;
apis: ApiItem[];
permissions: string[];
}
// the type of extension-samples.json file is SampleItem[]
```
## Example
Here is an example of the generated `extension-samples.json` file:
```json
[
{
"type": "API_SAMPLE",
"name": "alarms",
"title": "Alarms API Demo",
"description": "",
"repo_link": "https://github.com/GoogleChrome/chrome-extensions-samples/tree/main/api-samples/alarms",
"permissions": ["alarms"],
"apis": [
{
"type": "event",
"namespace": "runtime",
"propertyName": "onInstalled"
},
{
"type": "event",
"namespace": "action",
"propertyName": "onClicked"
},
{
"type": "event",
"namespace": "alarms",
"propertyName": "onAlarm"
},
{
"type": "type",
"namespace": "runtime",
"propertyName": "OnInstalledReason"
},
{
"type": "method",
"namespace": "alarms",
"propertyName": "create"
},
{
"type": "method",
"namespace": "tabs",
"propertyName": "create"
},
{
"type": "method",
"namespace": "alarms",
"propertyName": "clear"
},
{
"type": "method",
"namespace": "alarms",
"propertyName": "clearAll"
},
{
"type": "method",
"namespace": "alarms",
"propertyName": "getAll"
}
]
},
{
"type": "FUNCTIONAL_SAMPLE",
"name": "tutorial.getting-started",
"title": "Getting Started Example",
"description": "Build an Extension!",
"repo_link": "https://github.com/GoogleChrome/chrome-extensions-samples/tree/main/functional-samples/tutorial.getting-started",
"permissions": ["storage", "activeTab", "scripting"],
"apis": [
{
"type": "event",
"namespace": "runtime",
"propertyName": "onInstalled"
},
{
"type": "property",
"namespace": "storage",
"propertyName": "sync"
},
{
"type": "method",
"namespace": "tabs",
"propertyName": "query"
},
{
"type": "method",
"namespace": "scripting",
"propertyName": "executeScript"
}
]
}
]
```

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
{
"name": "sample-list-generator",
"version": "1.0.0",
"scripts": {
"start": "ts-node src/index.ts",
"prepare-chrome-types": "ts-node src/prepare-chrome-types.ts",
"test": "mocha --require ts-node/register test/**/*.test.ts"
},
"devDependencies": {
"@types/babel__core": "7.20.1",
"@types/mocha": "10.0.1",
"@types/node-fetch": "2.6.4",
"@types/sinon": "10.0.15",
"mocha": "10.2.0",
"sinon": "15.2.0",
"ts-node": "10.9.1",
"typescript": "5.1.3"
},
"dependencies": {
"@babel/core": "7.22.5",
"node-fetch": "2.6.11",
"typedoc": "0.24.8"
}
}

View File

@@ -0,0 +1,16 @@
export type FolderTypes = "API_SAMPLE" | "FUNCTIONAL_SAMPLE";
// Define all available folders for samples
export const AVAILABLE_FOLDERS: { path: string, type: FolderTypes }[] = [
{
path: 'api-samples',
type: 'API_SAMPLE'
},
{
path: 'functional-samples',
type: 'FUNCTIONAL_SAMPLE'
}
];
export const REPO_BASE_URL =
'https://github.com/GoogleChrome/chrome-extensions-samples/tree/main/';

View File

@@ -0,0 +1,15 @@
import path from 'path';
import fs from 'fs/promises';
import { getAllSamples } from './libs/sample-collector';
const start = async () => {
const samples = await getAllSamples();
// write to extension-samples.json
await fs.writeFile(
path.join(__dirname, '../extension-samples.json'),
JSON.stringify(samples, null, 2)
);
};
start();

View File

@@ -0,0 +1,198 @@
import {
ApiItem,
ApiItemWithType,
ApiTypeResult,
ExtensionApiMap
} from '../types';
import * as babel from '@babel/core';
import { isIdentifier } from '@babel/types';
import fs from 'fs/promises';
import { getAllJsFiles } from '../utils/filesystem';
import { loadExtensionApis } from './api-loader';
let EXTENSION_API_MAP: ExtensionApiMap = loadExtensionApis();
/**
* Gets the type of an api call.
* @param namespace - The namespace of the api call.
* @param propertyName - The property name of the api call.
* @returns The type of the api call.
* @example
* getApiType('tabs', 'query')
* // returns 'method'
*/
export const getApiType = (
namespace: string,
propertyName: string
): ApiTypeResult => {
namespace = namespace.replace(/_/g, '.');
const apiTypes = EXTENSION_API_MAP[namespace];
if (apiTypes) {
if (apiTypes.methods.includes(propertyName)) {
return 'method';
}
if (apiTypes.events.includes(propertyName)) {
return 'event';
}
if (apiTypes.properties.includes(propertyName)) {
return 'property';
}
if (apiTypes.types.includes(propertyName)) {
return 'type';
}
}
return 'unknown';
};
/**
* Gets all the api calls in a sample.
* @param sampleFolderPath - The path to the sample folder.
* @returns A promise that resolves to an array of apis the sample uses.
*/
export const getApiListForSample = async (
sampleFolderPath: string
): Promise<ApiItemWithType[]> => {
// get all js files in the folder
const jsFiles = await getAllJsFiles(sampleFolderPath);
const calls: ApiItemWithType[] = [];
await Promise.all(
jsFiles.map(async (file) => {
const callsFromFile = await extractApiCalls((await fs.readFile(file)).toString('utf-8'));
calls.push(...callsFromFile);
})
);
return uniqueItems(calls);
};
/**
* Gets the complete API call for the member expression.
* @param path - The path to the MemberExpression node.
* @returns The full member expression.
* @example
* getFullMemberExpression(path.node)
* // returns ['chrome', 'tabs', 'query']
*/
export function getFullMemberExpression(
path: babel.NodePath<babel.types.MemberExpression>
): string[] {
const result: string[] = [];
// Include the chrome. or browser. identifier
if (isIdentifier(path.node.object)) {
result.push(path.node.object.name);
} else {
// We don't support expressions
return result;
}
while (path) {
if (isIdentifier(path.node.property)) {
result.push(path.node.property.name);
} else {
// We don't support expressions
break;
}
const parentPath = path.parentPath;
if (!parentPath || !parentPath.isMemberExpression()) {
break;
} else {
path = parentPath;
}
}
return result;
}
/**
* Gets the namespace and property name of an api call.
* @param parts - The parts of the api call.
* @returns The namespace and property name of the api call.
* @example
* getApiItem(['chrome', 'tabs', 'query'])
* // returns { namespace: 'tabs', propertyName: 'query' }
* getApiItem(['chrome', 'devtools', 'inspectedWindow', 'eval'])
* // returns { namespace: 'devtools.inspectedWindow', propertyName: 'eval' }
*/
export function getApiItem(parts: string[]): ApiItem {
let namespace = '';
let propertyName = '';
// For some apis like `chrome.devtools.inspectedWindow.eval`,
// the namespace is actually `devtools.inspectedWindow`.
// So we need to check if the first two parts combined is a valid namespace.
if (EXTENSION_API_MAP[`${parts[0]}.${parts[1]}`]) {
namespace = `${parts[0]}.${parts[1]}`;
propertyName = parts[2];
} else {
namespace = parts[0];
propertyName = parts[1];
}
return { namespace, propertyName };
}
/**
* Filters an array of ApiItemWithType to remove duplicates.
* @param array - The array of ApiItemWithType to filter.
*/
function uniqueItems(array: ApiItemWithType[]) {
const tmp = new Set<string>();
return array.filter((item) => {
const fullApiString = `${item.namespace}.${item.propertyName}`;
return !tmp.has(fullApiString) && tmp.add(fullApiString);
});
}
/**
* Extracts all chrome and browser api calls from a file.
* @param script - The script string to extract api calls from.
* @returns A promise that resolves to an array of ApiItemWithType.
* @example
* extractApiCalls('chrome.tabs.query({})')
* // returns [{ type: 'method', namespace: 'tabs', propertyName: 'query' }]
*/
export const extractApiCalls = (script: string): Promise<ApiItemWithType[]> => {
return new Promise((resolve, reject) => {
const calls: ApiItemWithType[] = [];
babel.parse(
script,
{ ast: true, compact: false },
(err, result) => {
if (err || !result) {
reject(err);
return;
}
babel.traverse(result, {
MemberExpression(path) {
const parts = getFullMemberExpression(path);
// not a chrome or browser api
if (!['chrome', 'browser'].includes(parts.shift() || '')) {
return;
}
const { namespace, propertyName } = getApiItem(parts);
let type = getApiType(namespace, propertyName);
// api not found
if (type === 'unknown') {
console.warn('api not found', namespace, propertyName);
return;
}
calls.push({ type, namespace, propertyName });
}
});
resolve(calls);
}
);
});
};

View File

@@ -0,0 +1,19 @@
import path from 'path';
import fs from 'fs';
import type { ExtensionApiMap } from '../types';
import { isFileExistsSync } from '../utils/filesystem';
export const loadExtensionApis = (): ExtensionApiMap => {
const filePath = path.join(__dirname, '../../extension-apis.json');
// check if extension-apis.json exists
if (!isFileExistsSync(filePath)) {
console.error(
'extension-apis.json does not exist. Please run "npm run prepare-chrome-types" first.'
);
process.exit(1);
}
let data = fs.readFileSync(filePath, 'utf8');
return JSON.parse(data);
};

View File

@@ -0,0 +1,72 @@
import path from 'path';
import fs from 'fs/promises';
import { AVAILABLE_FOLDERS, REPO_BASE_URL } from '../constants';
import { getApiListForSample } from './api-detector';
import type { AvailableFolderItem, SampleItem } from '../types';
import { getBasePath, isDirectory, isFileExists } from '../utils/filesystem';
import { getManifest } from '../utils/manifest';
export const getAllSamples = async () => {
let samples: SampleItem[] = [];
// loop through all available folders
// e.g. api-samples, functional-samples
for (let samplesFolder of AVAILABLE_FOLDERS) {
const currentSamples = await getSamples(
samplesFolder.path,
samplesFolder.type
);
samples.push(...currentSamples);
}
return samples;
};
const getSamples = async (
currentRootFolderPath: string,
sampleType: AvailableFolderItem['type']
): Promise<SampleItem[]> => {
const samples: SampleItem[] = [];
const basePath = getBasePath();
// get all contents in the folder
const contents = await fs.readdir(path.join(basePath, currentRootFolderPath));
for (let content of contents) {
const currentPath = path.join(basePath, currentRootFolderPath, content);
// if content is not a folder, skip
if (!(await isDirectory(currentPath))) {
continue;
}
const manifestPath = path.join(currentPath, 'manifest.json');
// check if manifest.json exists
const manifestExists = await isFileExists(manifestPath);
if (manifestExists) {
// get manifest metadata
const manifestData = await getManifest(manifestPath);
// add to samples
samples.push({
type: sampleType,
name: content,
repo_link: new URL(
`${REPO_BASE_URL}${currentPath.replace(basePath, '')}`
).toString(),
apis: await getApiListForSample(currentPath),
title: manifestData.name || content,
description: manifestData.description || '',
permissions: manifestData.permissions || []
});
} else {
// if manifest.json does not exist, loop through all folders in current folder
const currentSamples = await getSamples(
path.join(currentRootFolderPath, content),
sampleType
);
samples.push(...currentSamples);
}
}
return samples;
};

View File

@@ -0,0 +1,73 @@
import fetch from 'node-fetch';
import path from 'path';
import fs from 'fs/promises';
import { ExtensionApiMap } from './types';
import { ReflectionKind } from 'typedoc';
// Fetch the latest version of the chrome types from storage
const fetchChromeTypes = async (): Promise<Record<string, any>> => {
console.log('Fetching chrome types...');
const response = await fetch(
'https://storage.googleapis.com/download/storage/v1/b/external-dcc-data/o/chrome-types.json?alt=media'
);
const chromeTypes = await response.json();
return chromeTypes;
};
const run = async () => {
const result: ExtensionApiMap = {};
const chromeTypes = await fetchChromeTypes();
for (const [chromeApiKey, chromeApiDetails] of Object.entries(chromeTypes)) {
const apiDetails: ExtensionApiMap[string] = {
properties: [],
methods: [],
types: [],
events: []
};
for (let property of chromeApiDetails._type.properties) {
const name = property.name as string;
// check property type
let propertyType = 'types';
if (property.kind & ReflectionKind.VariableOrProperty) {
propertyType = 'properties';
}
if (
property.type?.type === 'reference' &&
['CustomChromeEvent', 'events.Event', 'Event'].includes(
property.type.name
)
) {
propertyType = 'events';
}
if (property.signatures) {
propertyType = 'methods';
}
apiDetails[propertyType].push(name);
}
result[chromeApiKey] = apiDetails;
}
console.log('Writing to file...');
await fs.writeFile(
path.join(__dirname, '../extension-apis.json'),
JSON.stringify(
{
_comment:
'This file is autogenerated by running `npm run prepare-chrome-types`, do not edit.',
...result
},
null,
2
)
);
console.log('Done!');
};
run();

View File

@@ -0,0 +1,40 @@
import type { FolderTypes } from './constants';
export interface ApiItem {
namespace: string;
propertyName: string;
}
export interface ApiItemWithType extends ApiItem {
type: ApiTypeResult;
}
export interface ManifestData {
name: string;
description: string;
permissions: string[];
}
export type SampleItem = {
type: FolderTypes;
name: string;
repo_link: string;
apis: ApiItem[];
title: string;
description: string;
permissions: string[];
};
export interface AvailableFolderItem {
path: string;
type: FolderTypes;
}
export type ApiTypeResult =
| 'event'
| 'method'
| 'property'
| 'type'
| 'unknown';
export type ExtensionApiMap = Record<string, Record<string, string[]>>

View File

@@ -0,0 +1,53 @@
import fs from 'fs/promises';
import { accessSync } from 'fs';
import path from 'path';
export const getAllFiles = async (dir: string): Promise<string[]> => {
const result: string[] = [];
for (const file of await fs.readdir(dir)) {
const filePath = path.join(dir, file);
const stats = await fs.stat(filePath);
if (stats.isFile()) {
result.push(filePath);
} else if (stats.isDirectory()) {
result.push(...(await getAllFiles(filePath)));
}
}
return result;
};
export const getAllJsFiles = async (dir: string): Promise<string[]> => {
const allFiles = await getAllFiles(dir);
return allFiles.filter((file) =>
file.endsWith('.js')
);
}
export const isDirectory = async (path: string): Promise<boolean> => {
return (await fs.stat(path)).isDirectory()
}
export const isFileExists = async (filePath: string): Promise<boolean> => {
try {
await fs.access(filePath);
return true;
} catch {
return false;
}
};
export const isFileExistsSync = (filePath: string): boolean => {
try {
accessSync(filePath);
return true;
} catch {
return false;
}
};
export const getBasePath = (): string => {
return path.join(__dirname, '../../../../');
};

View File

@@ -0,0 +1,11 @@
import fs from 'fs/promises';
import { ManifestData } from '../types';
export const getManifest = async (
manifestPath: string
): Promise<ManifestData> => {
const manifest = await fs.readFile(manifestPath, 'utf8');
const parsedManifest = JSON.parse(manifest);
return parsedManifest;
};

View File

@@ -0,0 +1,207 @@
import { describe, it, beforeEach } from 'mocha';
import assert from 'assert';
import sinon from 'sinon';
import {
getApiType,
extractApiCalls,
getApiItem
} from '../../src/libs/api-detector';
describe('API Detector', function () {
beforeEach(function () {
sinon.reset();
});
describe('extractApiCalls()', function () {
it('should return correct api list for sample file (normal)', async function () {
const file = `
let a = 1;
let b = chrome.action.getBadgeText();
let c = chrome.action.setBadgeText(a);
chrome.action.onClicked.addListener(function (tab) {
console.log('clicked');
});
alert(chrome.contextMenus.ACTION_MENU_TOP_LEVEL_LIMIT)
`;
const result = await extractApiCalls(file);
assert.deepEqual(result, [
{
namespace: 'action',
propertyName: 'getBadgeText',
type: 'method'
},
{
namespace: 'action',
propertyName: 'setBadgeText',
type: 'method'
},
{
namespace: 'action',
propertyName: 'onClicked',
type: 'event'
},
{
namespace: 'contextMenus',
propertyName: 'ACTION_MENU_TOP_LEVEL_LIMIT',
type: 'property'
}
]);
});
it('should return correct api list for sample file (storage)', async function () {
const file = `
let b = await chrome.storage.local.get();
let c = await chrome.storage.sync.get();
let d = await chrome.storage.managed.get();
let e = await chrome.storage.session.get();
let f = await chrome.storage.onChanged.addListener();
`;
const result = await extractApiCalls(file);
assert.deepEqual(result, [
{
namespace: 'storage',
propertyName: 'local',
type: 'property'
},
{
namespace: 'storage',
propertyName: 'sync',
type: 'property'
},
{
namespace: 'storage',
propertyName: 'managed',
type: 'property'
},
{
namespace: 'storage',
propertyName: 'session',
type: 'property'
},
{
namespace: 'storage',
propertyName: 'onChanged',
type: 'event'
}
]);
});
it('should return correct api list for sample file (async)', async function () {
const file = `
let a = 1;
let b = await chrome.action.getBadgeText();
await chrome.action.setBadgeText(a);
`;
const result = await extractApiCalls(file);
assert.deepEqual(result, [
{
namespace: 'action',
propertyName: 'getBadgeText',
type: 'method'
},
{
namespace: 'action',
propertyName: 'setBadgeText',
type: 'method'
}
]);
});
it('should return correct api list for sample file (special case)', async function () {
const file = `
let a = 1;
let b = await chrome.system.cpu.getInfo();
chrome.devtools.network.onRequestFinished.addListener(
function(request) {
if (request.response.bodySize > 40*1024) {
chrome.devtools.inspectedWindow.eval(
'console.log("Large image: " + unescape("' +
escape(request.request.url) + '"))');
}
}
);
`;
const result = await extractApiCalls(file);
assert.deepEqual(result, [
{
namespace: 'system.cpu',
propertyName: 'getInfo',
type: 'method'
},
{
namespace: 'devtools.network',
propertyName: 'onRequestFinished',
type: 'event'
},
{
namespace: 'devtools.inspectedWindow',
propertyName: 'eval',
type: 'method'
}
]);
});
});
describe('getApiType()', function () {
it('should return correct type of api in normal case', function () {
let apiType = getApiType('action', 'getBadgeText');
assert.equal(apiType, 'method');
});
it('should return correct type of api in special case', function () {
let apiType = getApiType('devtools.network', 'onNavigated');
assert.equal(apiType, 'event');
});
it('should return unknown when api not found', function () {
let apiType = getApiType('action', '123');
assert.equal(apiType, 'unknown');
});
});
describe('getApiItem()', function () {
it('should return correct api item', function () {
let apiItem = getApiItem(['action', 'getBadgeText']);
assert.deepEqual(apiItem, {
namespace: 'action',
propertyName: 'getBadgeText'
});
});
it('should return correct api item (storage)', function () {
let apiItem = getApiItem(['storage', 'sync', 'get']);
assert.deepEqual(apiItem, {
namespace: 'storage',
propertyName: 'sync'
});
apiItem = getApiItem(['storage', 'sync', 'onChanged']);
assert.deepEqual(apiItem, {
namespace: 'storage',
propertyName: 'sync'
});
apiItem = getApiItem(['storage', 'onChanged']);
assert.deepEqual(apiItem, {
namespace: 'storage',
propertyName: 'onChanged'
});
});
it('should return correct api item (special case)', function () {
let apiItem = getApiItem([
'devtools',
'network',
'onRequestFinished',
'addListener'
]);
assert.deepEqual(apiItem, {
namespace: 'devtools.network',
propertyName: 'onRequestFinished'
});
});
});
});