Merge pull request #1542 from 0xProject/feature/solc-docker

Implement docker as another backend for sol-compiler

16 files changed, 167 insertions, 39 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index af25def64..0aec168cf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -17,6 +17,7 @@ jobs:
             - run:
                   name: yarn
                   command: yarn --frozen-lockfile --ignore-engines install
+            - setup_remote_docker
             - run: yarn build:ci:no_website
             - run: yarn build:ts
             - save_cache:
diff --git a/contracts/asset-proxy/compiler.json b/contracts/asset-proxy/compiler.json
index 6a077e965..70d4c6b20 100644
--- a/contracts/asset-proxy/compiler.json
+++ b/contracts/asset-proxy/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/erc20/compiler.json b/contracts/erc20/compiler.json
index 58fc8ea9b..264f6231a 100644
--- a/contracts/erc20/compiler.json
+++ b/contracts/erc20/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/erc721/compiler.json b/contracts/erc721/compiler.json
index c5f07ddb8..db8589016 100644
--- a/contracts/erc721/compiler.json
+++ b/contracts/erc721/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/exchange-forwarder/compiler.json b/contracts/exchange-forwarder/compiler.json
index b0237a519..e93eb4bdd 100644
--- a/contracts/exchange-forwarder/compiler.json
+++ b/contracts/exchange-forwarder/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/exchange-libs/compiler.json b/contracts/exchange-libs/compiler.json
index 349d3063b..c4f0c6bd9 100644
--- a/contracts/exchange-libs/compiler.json
+++ b/contracts/exchange-libs/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/exchange/compiler.json b/contracts/exchange/compiler.json
index e53fce0ec..6a5f666d1 100644
--- a/contracts/exchange/compiler.json
+++ b/contracts/exchange/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json
index 1dfaf7a05..91e0bd429 100644
--- a/contracts/extensions/compiler.json
+++ b/contracts/extensions/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json
index 772665775..ae3905238 100644
--- a/contracts/multisig/compiler.json
+++ b/contracts/multisig/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json
index 1524c1eaa..7473ea6d1 100644
--- a/contracts/utils/compiler.json
+++ b/contracts/utils/compiler.json
@@ -1,6 +1,7 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,
diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts
index a8dcfd68a..8339ab5a6 100644
--- a/packages/ethereum-types/src/index.ts
+++ b/packages/ethereum-types/src/index.ts
@@ -509,6 +509,7 @@ export interface Source {
  * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description)
  * contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the
  * specified directory.
+ * useDockerisedSolc: If set to true - sol-compiler will try calling a dockerized installations of solc to achieve faster compilation times. Otherwise and by default - solcjs will be used. Defaults to false.
  * solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all
  * contracts to compile with the the version specified here.
  */
@@ -517,5 +518,6 @@ export interface CompilerOptions {
     artifactsDir?: string;
     compilerSettings?: CompilerSettings;
     contracts?: string[] | '*';
+    useDockerisedSolc?: boolean;
     solcVersion?: string;
 } // tslint:disable-line:max-file-line-count
diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts
index d38ccbf39..efee3eb8a 100644
--- a/packages/sol-compiler/src/compiler.ts
+++ b/packages/sol-compiler/src/compiler.ts
@@ -10,6 +10,7 @@ import {
     URLResolver,
 } from '@0x/sol-resolver';
 import { logUtils } from '@0x/utils';
+import { execSync } from 'child_process';
 import * as chokidar from 'chokidar';
 import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types';
 import * as fs from 'fs';
@@ -23,13 +24,16 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema';
 import { binPaths } from './solc/bin_paths';
 import {
     addHexPrefixToContractBytecode,
-    compile,
+    compileDockerAsync,
+    compileSolcJSAsync,
     createDirIfDoesNotExistAsync,
     getContractArtifactIfExistsAsync,
-    getSolcAsync,
+    getDependencyNameToPackagePath,
     getSourcesWithDependencies,
     getSourceTreeHash,
+    makeContractPathsRelative,
     parseSolidityVersionRange,
+    printCompilationErrorsAndWarnings,
 } from './utils/compiler';
 import { constants } from './utils/constants';
 import { fsWrapper } from './utils/fs_wrapper';
@@ -40,6 +44,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*';
 const ALL_FILES_IDENTIFIER = '*';
 const DEFAULT_CONTRACTS_DIR = path.resolve('contracts');
 const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts');
+const DEFAULT_USE_DOCKERISED_SOLC = false;
 // Solc compiler settings cannot be configured from the commandline.
 // If you need this configured, please create a `compiler.json` config file
 // with your desired configurations.
@@ -84,6 +89,7 @@ export class Compiler {
     private readonly _artifactsDir: string;
     private readonly _solcVersionIfExists: string | undefined;
     private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
+    private readonly _useDockerisedSolc: boolean;
     /**
      * Instantiates a new instance of the Compiler class.
      * @param opts Optional compiler options
@@ -97,16 +103,17 @@ export class Compiler {
             : {};
         const passedOpts = opts || {};
         assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema);
-        this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR;
+        this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR);
         this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion;
         this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS;
         this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR;
         this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER;
-        this._nameResolver = new NameResolver(path.resolve(this._contractsDir));
+        this._useDockerisedSolc =
+            passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC;
+        this._nameResolver = new NameResolver(this._contractsDir);
         const resolver = new FallthroughResolver();
         resolver.appendResolver(new URLResolver());
-        const packagePath = path.resolve('');
-        resolver.appendResolver(new NPMResolver(packagePath));
+        resolver.appendResolver(new NPMResolver(this._contractsDir));
         resolver.appendResolver(new RelativeFSResolver(this._contractsDir));
         resolver.appendResolver(new FSResolver());
         resolver.appendResolver(this._nameResolver);
@@ -206,10 +213,12 @@ export class Compiler {
         // map contract paths to data about them for later verification and persistence
         const contractPathToData: ContractPathToData = {};
 
+        const resolvedContractSources = [];
         for (const contractName of contractNames) {
-            const contractSource = this._resolver.resolve(contractName);
+            const spyResolver = new SpyResolver(this._resolver);
+            const contractSource = spyResolver.resolve(contractName);
             const sourceTreeHashHex = getSourceTreeHash(
-                this._resolver,
+                spyResolver,
                 path.join(this._contractsDir, contractSource.path),
             ).toString('hex');
             const contractData = {
@@ -236,26 +245,54 @@ export class Compiler {
                 };
             }
             // add input to the right version batch
-            versionToInputs[solcVersion].standardInput.sources[contractSource.path] = {
-                content: contractSource.source,
-            };
+            for (const resolvedContractSource of spyResolver.resolvedContractSources) {
+                versionToInputs[solcVersion].standardInput.sources[resolvedContractSource.absolutePath] = {
+                    content: resolvedContractSource.source,
+                };
+            }
+            resolvedContractSources.push(...spyResolver.resolvedContractSources);
             versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
         }
 
-        const compilerOutputs: StandardOutput[] = [];
+        const dependencyNameToPath = getDependencyNameToPackagePath(resolvedContractSources);
 
-        const solcVersions = _.keys(versionToInputs);
-        for (const solcVersion of solcVersions) {
+        const compilerOutputs: StandardOutput[] = [];
+        for (const solcVersion of _.keys(versionToInputs)) {
             const input = versionToInputs[solcVersion];
             logUtils.warn(
                 `Compiling ${input.contractsToCompile.length} contracts (${
                     input.contractsToCompile
                 }) with Solidity v${solcVersion}...`,
             );
-
-            const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion);
-            const compilerOutput = compile(this._resolver, solcInstance, input.standardInput);
-            compilerOutputs.push(compilerOutput);
+            let compilerOutput;
+            let fullSolcVersion;
+            input.standardInput.settings.remappings = _.map(
+                dependencyNameToPath,
+                (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`,
+            );
+            if (this._useDockerisedSolc) {
+                const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`;
+                const versionCommandOutput = execSync(dockerCommand).toString();
+                const versionCommandOutputParts = versionCommandOutput.split(' ');
+                fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim();
+                compilerOutput = await compileDockerAsync(solcVersion, input.standardInput);
+            } else {
+                fullSolcVersion = binPaths[solcVersion];
+                compilerOutput = await compileSolcJSAsync(solcVersion, input.standardInput);
+            }
+            if (!_.isUndefined(compilerOutput.errors)) {
+                printCompilationErrorsAndWarnings(compilerOutput.errors);
+            }
+            compilerOutput.sources = makeContractPathsRelative(
+                compilerOutput.sources,
+                this._contractsDir,
+                dependencyNameToPath,
+            );
+            compilerOutput.contracts = makeContractPathsRelative(
+                compilerOutput.contracts,
+                this._contractsDir,
+                dependencyNameToPath,
+            );
 
             for (const contractPath of input.contractsToCompile) {
                 const contractName = contractPathToData[contractPath].contractName;
@@ -280,6 +317,8 @@ export class Compiler {
                     );
                 }
             }
+
+            compilerOutputs.push(compilerOutput);
         }
 
         return compilerOutputs;
diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts
index d4d1b6017..c0766b625 100644
--- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts
+++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts
@@ -19,6 +19,7 @@ export const compilerOptionsSchema = {
                 },
             ],
         },
+        useDockerisedSolc: { type: 'boolean' },
     },
     type: 'object',
     required: [],
diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts
index db308f2b5..c75f76dac 100644
--- a/packages/sol-compiler/src/utils/compiler.ts
+++ b/packages/sol-compiler/src/utils/compiler.ts
@@ -1,6 +1,7 @@
 import { ContractSource, Resolver } from '@0x/sol-resolver';
 import { fetchAsync, logUtils } from '@0x/utils';
 import chalk from 'chalk';
+import { execSync } from 'child_process';
 import { ContractArtifact } from 'ethereum-types';
 import * as ethUtil from 'ethereumjs-util';
 import * as _ from 'lodash';
@@ -117,31 +118,74 @@ export function parseDependencies(contractSource: ContractSource): string[] {
 
 /**
  * Compiles the contracts and prints errors/warnings
- * @param resolver Resolver
- * @param solcInstance Instance of a solc compiler
+ * @param solcVersion Version of a solc compiler
  * @param standardInput Solidity standard JSON input
  */
-export function compile(
-    resolver: Resolver,
-    solcInstance: solc.SolcInstance,
+export async function compileSolcJSAsync(
+    solcVersion: string,
     standardInput: solc.StandardInput,
-): solc.StandardOutput {
+): Promise<solc.StandardOutput> {
+    const solcInstance = await getSolcJSAsync(solcVersion);
     const standardInputStr = JSON.stringify(standardInput);
-    const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr, importPath => {
-        const sourceCodeIfExists = resolver.resolve(importPath);
-        return { contents: sourceCodeIfExists.source };
-    });
+    const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr);
+    const compiled: solc.StandardOutput = JSON.parse(standardOutputStr);
+    return compiled;
+}
+
+/**
+ * Compiles the contracts and prints errors/warnings
+ * @param solcVersion Version of a solc compiler
+ * @param standardInput Solidity standard JSON input
+ */
+export async function compileDockerAsync(
+    solcVersion: string,
+    standardInput: solc.StandardInput,
+): Promise<solc.StandardOutput> {
+    const standardInputStr = JSON.stringify(standardInput, null, 2);
+    const dockerCommand = `docker run -i -a stdin -a stdout -a stderr ethereum/solc:${solcVersion} solc --standard-json`;
+    const standardOutputStr = execSync(dockerCommand, { input: standardInputStr }).toString();
     const compiled: solc.StandardOutput = JSON.parse(standardOutputStr);
-    if (!_.isUndefined(compiled.errors)) {
-        printCompilationErrorsAndWarnings(compiled.errors);
-    }
     return compiled;
 }
+
+/**
+ * Example "relative" paths:
+ * /user/leo/0x-monorepo/contracts/extensions/contracts/extension.sol -> extension.sol
+ * /user/leo/0x-monorepo/node_modules/@0x/contracts-protocol/contracts/exchange.sol -> @0x/contracts-protocol/contracts/exchange.sol
+ */
+function makeContractPathRelative(
+    absolutePath: string,
+    contractsDir: string,
+    dependencyNameToPath: { [dependencyName: string]: string },
+): string {
+    let contractPath = absolutePath.replace(`${contractsDir}/`, '');
+    _.map(dependencyNameToPath, (packagePath: string, dependencyName: string) => {
+        contractPath = contractPath.replace(packagePath, dependencyName);
+    });
+    return contractPath;
+}
+
+/**
+ * Makes the path relative removing all system-dependent data. Converts absolute paths to a format suitable for artifacts.
+ * @param absolutePathToSmth Absolute path to contract or source
+ * @param contractsDir Current package contracts directory location
+ * @param dependencyNameToPath Mapping of dependency name to package path
+ */
+export function makeContractPathsRelative(
+    absolutePathToSmth: { [absoluteContractPath: string]: any },
+    contractsDir: string,
+    dependencyNameToPath: { [dependencyName: string]: string },
+): { [contractPath: string]: any } {
+    return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) =>
+        makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPath),
+    );
+}
+
 /**
  * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning).
  * @param solcErrors The errors field of standard JSON output that contains errors and warnings.
  */
-function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void {
+export function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void {
     const SOLIDITY_WARNING = 'warning';
     const errors = _.filter(solcErrors, entry => entry.severity !== SOLIDITY_WARNING);
     const warnings = _.filter(solcErrors, entry => entry.severity === SOLIDITY_WARNING);
@@ -267,13 +311,11 @@ function recursivelyGatherDependencySources(
 }
 
 /**
- * Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS,
+ * Gets the solidity compiler instance. If the compiler is already cached - gets it from FS,
  * otherwise - fetches it and caches it.
  * @param solcVersion The compiler version. e.g. 0.5.0
  */
-export async function getSolcAsync(
-    solcVersion: string,
-): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
+export async function getSolcJSAsync(solcVersion: string): Promise<solc.SolcInstance> {
     const fullSolcVersion = binPaths[solcVersion];
     if (_.isUndefined(fullSolcVersion)) {
         throw new Error(`${solcVersion} is not a known compiler version`);
@@ -297,7 +339,7 @@ export async function getSolcAsync(
         throw new Error('No compiler available');
     }
     const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
-    return { solcInstance, fullSolcVersion };
+    return solcInstance;
 }
 
 /**
@@ -319,3 +361,35 @@ export function addHexPrefixToContractBytecode(compiledContract: solc.StandardCo
         }
     }
 }
+
+/**
+ * Takes the list of resolved contract sources from `SpyResolver` and produces a mapping from dependency name
+ * to package path used in `remappings` later, as well as in generating the "relative" source paths saved to the artifact files.
+ * @param contractSources The list of resolved contract sources
+ */
+export function getDependencyNameToPackagePath(
+    contractSources: ContractSource[],
+): { [dependencyName: string]: string } {
+    const allTouchedFiles = contractSources.map(contractSource => `${contractSource.absolutePath}`);
+    const NODE_MODULES = 'node_modules';
+    const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES));
+    const dependencyNameToPath: { [dependencyName: string]: string } = {};
+    _.map(allTouchedDependencies, dependencyFilePath => {
+        const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES);
+        const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length;
+        const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1);
+        let packageName;
+        let packageScopeIfExists;
+        let dependencyName;
+        if (_.startsWith(importPath, '@')) {
+            [packageScopeIfExists, packageName] = importPath.split('/');
+            dependencyName = `${packageScopeIfExists}/${packageName}`;
+        } else {
+            [packageName] = importPath.split('/');
+            dependencyName = `${packageName}`;
+        }
+        const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName);
+        dependencyNameToPath[dependencyName] = dependencyPackagePath;
+    });
+    return dependencyNameToPath;
+}
diff --git a/packages/sol-resolver/src/resolvers/fs_resolver.ts b/packages/sol-resolver/src/resolvers/fs_resolver.ts
index 86128023d..248fa405e 100644
--- a/packages/sol-resolver/src/resolvers/fs_resolver.ts
+++ b/packages/sol-resolver/src/resolvers/fs_resolver.ts
@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import * as path from 'path';
 
 import { ContractSource } from '../types';
 
@@ -9,7 +10,8 @@ export class FSResolver extends Resolver {
     public resolveIfExists(importPath: string): ContractSource | undefined {
         if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) {
             const fileContent = fs.readFileSync(importPath).toString();
-            return { source: fileContent, path: importPath, absolutePath: importPath };
+            const absolutePath = path.resolve(importPath);
+            return { source: fileContent, path: importPath, absolutePath } as any;
         }
         return undefined;
     }
diff --git a/packages/typescript-typings/types/solc/index.d.ts b/packages/typescript-typings/types/solc/index.d.ts
index f4c05cd7c..fefad9f6a 100644
--- a/packages/typescript-typings/types/solc/index.d.ts
+++ b/packages/typescript-typings/types/solc/index.d.ts
@@ -95,7 +95,7 @@ declare module 'solc' {
             optimizerEnabled: number,
             findImports: (importPath: string) => ImportContents,
         ): CompilationResult;
-        compileStandardWrapper(input: string, findImports: (importPath: string) => ImportContents): string;
+        compileStandardWrapper(input: string, findImports?: (importPath: string) => ImportContents): string;
     }
     export function loadRemoteVersion(
         versionName: string,