OSV data source (#22)

* OSV source

* Added tests

* Added tests

* Added tests

Author: prabhu

README.md

@@ -1,6 +1,6 @@
 # Introduction
 
-This repo is a vulnerability database and package search for sources such as NVD, GitHub and so on. It uses a built-in file based storage to allow offline access.
+This repo is a vulnerability database and package search for sources such as OSV, NVD, GitHub, and NPM. Vulnerability data are downloaded from the sources and stored in a custom file based storage with indexes to allow offline access and quick searches.
 
 ## Installation
 
@@ -14,10 +14,18 @@ This package is ideal as a library for managing vulnerabilities. This is used by
 
 ### Cache vulnerability data
 
+Cache from all sources
+
 ```bash
 vdb --cache
 ```
 
+Cache from just [OSV](https://osv.dev)
+
+```bash
+vdb --cache --only-osv
+```
+
 It is possible to customise the cache behaviour by increasing the historic data period to cache by setting the following environment variables.
 
 - NVD_START_YEAR - Default: 2016. Supports upto 2002

requirements.txt

@@ -4,3 +4,4 @@ tabulate
 msgpack
 orjson
 semver
+packageurl-python

setup.py

@@ -5,16 +5,16 @@
 
 setuptools.setup(
     name="appthreat-vulnerability-db",
-    version="1.7.2",
+    version="2.0.0",
     author="Team AppThreat",
     author_email="[email protected]",
-    description="AppThreat's vulnerability database and package search library with a built-in file based storage. CVE, GitHub, npm are the primary sources of vulnerabilities.",
+    description="AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities.",
     entry_points={"console_scripts": ["vdb=vdb.cli:main"]},
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/appthreat/vulnerability-db",
     packages=setuptools.find_packages(),
-    install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson", "semver"],
+    install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson", "semver", "packageurl-python"],
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",

test/data/osv_mixed_data.json

@@ -0,0 +1 @@
+{"affected":[{"database_specific":{"cvss":{"score":7.3,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L"},"cwes":[{"cweId":"CWE-843","description":"The program allocates or initializes a resource such as a pointer, object, or variable using one type, but it later accesses that resource using a type that is incompatible with the original type.","name":"Access of Resource Using Incompatible Type ('Type Confusion')"},{"cweId":"CWE-1321","description":"The software receives input from an upstream component that specifies attributes that are to be initialized or updated in an object, but it does not properly control modifications of attributes of the object prototype.","name":"Improperly Controlled Modification of Object Prototype Attributes ('Prototype Pollution')"}],"ghsa":"https://github.com/advisories/GHSA-4jqc-8m5r-9rpr"},"package":{"ecosystem":"npm","name":"set-value"},"ranges":[{"events":[{"introduced":"0"},{"fixed":"2.0.1"},{"introduced":"3.0.0"},{"fixed":"4.0.1"}],"id":0,"type":"SEMVER"}]},{"database_specific":{"cvss":{"score":7.3,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L"},"cwes":[{"cweId":"CWE-843","description":"The program allocates or initializes a resource such as a pointer, object, or variable using one type, but it later accesses that resource using a type that is incompatible with the original type.","name":"Access of Resource Using Incompatible Type ('Type Confusion')"},{"cweId":"CWE-1321","description":"The software receives input from an upstream component that specifies attributes that are to be initialized or updated in an object, but it does not properly control modifications of attributes of the object prototype.","name":"Improperly Controlled Modification of Object Prototype Attributes ('Prototype Pollution')"}],"ghsa":"https://github.com/advisories/GHSA-4jqc-8m5r-9rpr"},"package":{"ecosystem":"NuGet","name":"set-value-nuget"},"ranges":[{"events":[{"introduced":"0"},{"fixed":"2.0.0"}],"id":0,"type":"ECOSYSTEM"}]}],"aliases":["CVE-2021-23440"],"details":"This affects the package set-value before 4.0.1. A type confusion vulnerability can lead to a bypass of CVE-2019-10747 when the user-provided keys used in the path parameter are arrays.","id":"GHSA-4jqc-8m5r-9rpr","invalid":false,"isFixed":true,"modified":"2021-11-12T22:48:18Z","published":"2021-09-13T20:09:36Z","references":[{"type":"ADVISORY","url":"https://nvd.nist.gov/vuln/detail/CVE-2021-23440"},{"type":"WEB","url":"https://github.com/jonschlinkert/set-value/pull/33"},{"type":"WEB","url":"https://github.com/jonschlinkert/set-value/commit/7cf8073bb06bf0c15e08475f9f952823b4576452"},{"type":"WEB","url":"https://snyk.io/vuln/SNYK-JAVA-ORGWEBJARSNPM-1584212"},{"type":"WEB","url":"https://snyk.io/vuln/SNYK-JS-SETVALUE-1540541"},{"type":"WEB","url":"https://www.huntr.dev/bounties/2eae1159-01de-4f82-a177-7478a408c4a2/"},{"type":"ADVISORY","url":"https://github.com/advisories/GHSA-4jqc-8m5r-9rpr"}],"source":"https://storage.googleapis.com/ghsa-osv/GHSA-4jqc-8m5r-9rpr.json","source_link":"https://storage.googleapis.com/ghsa-osv/GHSA-4jqc-8m5r-9rpr.json","summary":"Prototype Pollution in set-value"}

test/data/osv_mvn_data.json

@@ -0,0 +1 @@
+{"affected":[{"database_specific":{"cvss":{"score":7.0,"vectorString":"CVSS:3.0/AV:L/AC:H/PR:L/UI:N/S:U/C:H/I:H/A:H"},"cwes":[{"cweId":"CWE-470","description":"The application uses external input with reflection to select which classes or code to use, but it does not sufficiently prevent the input from selecting improper classes or code.","name":"Use of Externally-Controlled Input to Select Classes or Code ('Unsafe Reflection')"}],"ghsa":"https://github.com/advisories/GHSA-xxgp-pcfc-3vgc"},"package":{"ecosystem":"Maven","name":"org.hibernate:hibernate-validator"},"ranges":[{"events":[{"introduced":"5.4.0"},{"fixed":"5.4.2"},{"introduced":"5.3.0"},{"fixed":"5.3.6"},{"introduced":"5.2.0"},{"fixed":"5.2.5"}],"id":0,"type":"ECOSYSTEM"}],"versions":["5.2.0.Final","5.2.1.Final","5.2.2.Final","5.2.3.Final","5.2.4.Final","5.3.0.Final","5.3.1.Final","5.3.2.Final","5.3.3.Final","5.3.4.Final","5.3.5.Final","5.4.0.Final","5.4.1.Final"]}],"aliases":["CVE-2017-7536"],"details":"In Hibernate Validator 5.2.x before 5.2.5 final, 5.3.x, and 5.4.x, it was found that when the security manager's reflective permissions, which allows it to access the private members of the class, are granted to Hibernate Validator, a potential privilege escalation can occur. By allowing the calling code to access those private members without the permission an attacker may be able to validate an invalid instance and access the private member value via ConstraintViolation#getInvalidValue().","id":"GHSA-xxgp-pcfc-3vgc","invalid":false,"isFixed":true,"modified":"2021-12-13T23:44:21.151108Z","published":"2020-06-15T19:57:48Z","references":[{"type":"ADVISORY","url":"https://nvd.nist.gov/vuln/detail/CVE-2017-7536"},{"type":"WEB","url":"https://bugzilla.redhat.com/show_bug.cgi?id=1465573"},{"type":"WEB","url":"https://lists.apache.org/thread.html/9317fd092b257a0815434b116a8af8daea6e920b6673f4fd5583d5fe@%3Ccommits.druid.apache.org%3E"},{"type":"WEB","url":"http://www.securityfocus.com/bid/101048"},{"type":"WEB","url":"http://www.securitytracker.com/id/1039744"},{"type":"WEB","url":"https://github.com/hibernate/hibernate-validator/commit/0778a5c98b817771a645c6f4ba0b28dd8b5437b"},{"type":"WEB","url":"https://github.com/hibernate/hibernate-validator/commit/0886e89900d343ea20fde5137c9a3086e6da9ac"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:2808"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:2809"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:2810"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:2811"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:3141"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:3454"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:3455"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:3456"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2017:3458"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:2740"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:2741"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:2742"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:2743"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:2927"},{"type":"WEB","url":"https://access.redhat.com/errata/RHSA-2018:3817"},{"type":"ADVISORY","url":"https://github.com/advisories/GHSA-xxgp-pcfc-3vgc"}],"source":"https://storage.googleapis.com/ghsa-osv/GHSA-xxgp-pcfc-3vgc.json","source_link":"https://storage.googleapis.com/ghsa-osv/GHSA-xxgp-pcfc-3vgc.json","summary":"Privilege Escalation in Hibernate Validator"}

test/data/osv_rust_data.json

@@ -0,0 +1 @@
+{"id":"RUSTSEC-2020-0014","summary":"Various memory safety issues","details":"Several memory safety issues have been uncovered in an audit of\nrusqlite.\n\nSee https://github.com/rusqlite/rusqlite/releases/tag/0.23.0 for a complete list.","aliases":["CVE-2020-35866","CVE-2020-35867","CVE-2020-35868","CVE-2020-35869","CVE-2020-35870","CVE-2020-35871","CVE-2020-35872","CVE-2020-35873"],"modified":"2021-01-04T17:02:59Z","published":"2020-04-23T12:00:00Z","references":[{"type":"PACKAGE","url":"https://crates.io/crates/rusqlite"},{"type":"ADVISORY","url":"https://rustsec.org/advisories/RUSTSEC-2020-0014.html"},{"type":"WEB","url":"https://github.com/rusqlite/rusqlite/releases/tag/0.23.0"}],"affected":[{"package":{"name":"rusqlite","ecosystem":"crates.io","purl":"pkg:cargo/rusqlite"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0.0.0-0"},{"fixed":"0.23.0"}]}],"ecosystem_specific":{"affects":{"arch":[],"os":[],"functions":["rusqlite::Connection::get_aux","rusqlite::Connection::set_aux","rusqlite::session::Session::attach","rusqlite::session::Session::diff","rusqlite::trace::log","rusqlite::vtab::create_module"]}},"database_specific":{"categories":[],"informational":null,"source":"https://github.com/rustsec/advisory-db/blob/osv/crates/RUSTSEC-2020-0014.json","cvss":null}}]}

test/test_source.py

@@ -5,6 +5,7 @@
 
 from vdb.lib.gha import GitHubSource
 from vdb.lib.nvd import NvdSource
+from vdb.lib.osv import OSVSource
 
 
 @pytest.fixture
@@ -25,6 +26,33 @@ def test_cve_wconfig_json():
         return json.loads(fp.read())
 
 
+@pytest.fixture
+def test_osv_rust_json():
+    test_cve_data = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "data", "osv_rust_data.json"
+    )
+    with open(test_cve_data, "r") as fp:
+        return json.loads(fp.read())
+
+
+@pytest.fixture
+def test_osv_mvn_json():
+    test_cve_data = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "data", "osv_mvn_data.json"
+    )
+    with open(test_cve_data, "r") as fp:
+        return json.loads(fp.read())
+
+
+@pytest.fixture
+def test_osv_mixed_json():
+    test_cve_data = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "data", "osv_mixed_data.json"
+    )
+    with open(test_cve_data, "r") as fp:
+        return json.loads(fp.read())
+
+
 def test_convert(test_cve_json):
     nvdlatest = NvdSource()
     data = nvdlatest.convert(test_cve_json)
@@ -102,3 +130,24 @@ def test_gha_version_ranges():
     assert version_list == ("1.1.0", "", "", "")
     version_list = source.get_version_range("> 11.0, < 24.1.1")
     assert version_list == ("", "", "11.0", "24.1.1")
+
+
+@pytest.mark.skip(reason="This downloads and tests with live data")
+def test_osv_download_all():
+    osvlatest = OSVSource()
+    data = osvlatest.download_all()
+    assert len(data) > 1000
+
+
+def test_osv_convert(test_osv_rust_json, test_osv_mvn_json, test_osv_mixed_json):
+    osvlatest = OSVSource()
+    cve_data = osvlatest.convert(test_osv_rust_json)
+    assert cve_data
+
+    cve_data = osvlatest.convert(test_osv_mvn_json)
+    assert cve_data
+    assert len(cve_data) == 3
+
+    cve_data = osvlatest.convert(test_osv_mixed_json)
+    assert cve_data
+    assert len(cve_data) == 3

vdb/cli.py

@@ -13,6 +13,7 @@
 from vdb.lib.gha import GitHubSource
 from vdb.lib.npm import NpmSource
 from vdb.lib.nvd import NvdSource
+from vdb.lib.osv import OSVSource
 
 logging.basicConfig(
     level=logging.INFO, format="%(levelname)s [%(asctime)s] %(message)s"
@@ -52,6 +53,13 @@ def build_args():
         dest="cache",
         help="Cache vulnerability information in platform specific user_data_dir",
     )
+    parser.add_argument(
+        "--only-osv",
+        action="store_true",
+        default=False,
+        dest="only_osv",
+        help="Use only OSV as the source.",
+    )
     parser.add_argument(
         "--sync",
         action="store_true",
@@ -146,11 +154,12 @@ def main():
                 os.rmdir(config.data_dir)
             except Exception:
                 pass
-    else:
-        LOG.info("Vulnerability database loaded from {}".format(config.vdb_bin_file))
-
     if args.cache:
-        for s in [GitHubSource(), NvdSource()]:
+        if args.only_osv:
+            sources = [OSVSource()]
+        else:
+            sources = [OSVSource(), GitHubSource(), NvdSource()]
+        for s in sources:
             LOG.info("Refreshing {}".format(s.__class__.__name__))
             s.refresh()
     elif args.sync:
@@ -174,6 +183,7 @@ def main():
         results = dbLib.list_all_occurrence(db)
         print_results(results)
     elif args.search:
+        LOG.info("Vulnerability database loaded from {}".format(config.vdb_bin_file))
         db = dbLib.get()
         search_list = re.split(r"[,|;]", args.search)
         for pkg_info in search_list:

vdb/lib/__init__.py

@@ -23,9 +23,9 @@
     "npm": ["javascript", "node.js", "nodejs"],
     "nuget": [".net_framework", "csharp", ".net_core"],
     "pypi": ["python"],
-    "rubygems": ["ruby"],
+    "rubygems": ["ruby", "gem"],
     "golang": ["go"],
-    "crates": ["rust"],
+    "crates": ["rust", "crates.io", "cargo"],
 }
 
 # CPE Regex

vdb/lib/config.py

@@ -5,15 +5,15 @@
 # NVD CVE json feed url
 nvd_url = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%(year)s.json.gz"
 
-# NVD start year. 2016 is quicker. 2002 is quite detailed but slow
-nvd_start_year = os.getenv("NVD_START_YEAR", 2016)
+# NVD start year. 2018 is quicker. 2002 is quite detailed but slow
+nvd_start_year = os.getenv("NVD_START_YEAR", 2018)
 
 # GitHub advisory feed url
 gha_url = os.getenv("GITHUB_GRAPHQL_URL", "https://api.github.com/graphql")
 
 # No of pages to download from GitHub during a full refresh
-gha_pages_count = os.getenv("GITHUB_PAGE_COUNT", 10)
-npm_pages_count = os.getenv("NPM_PAGE_COUNT", 10)
+gha_pages_count = os.getenv("GITHUB_PAGE_COUNT", 2)
+npm_pages_count = os.getenv("NPM_PAGE_COUNT", 2)
 
 # DB file dir
 data_dir = os.getenv("VDB_HOME", user_data_dir("vdb"))
@@ -37,6 +37,15 @@
 {"cve":{"data_type":"CVE","data_format":"MITRE","data_version":"4.0","CVE_data_meta":{"ID":"%(cve_id)s","ASSIGNER":"%(assigner)s"},"problemtype":{"problemtype_data":[{"description":[{"lang":"en","value":"%(cwe_id)s"}]}]},"references":{"reference_data": %(references)s},"description":{"description_data":[{"lang":"en","value":"%(description)s"}]}},"configurations":{"CVE_data_version":"4.0","nodes":[{"operator":"OR","cpe_match":[{"vulnerable":true,"cpe23Uri":"cpe:2.3:a:%(vendor)s:%(product)s:%(version)s:*:*:*:*:*:*:*","versionStartExcluding":"%(version_start_excluding)s","versionEndExcluding":"%(version_end_excluding)s","versionStartIncluding":"%(version_start_including)s","versionEndIncluding":"%(version_end_including)s"}, {"vulnerable":false,"cpe23Uri":"cpe:2.3:a:%(vendor)s:%(product)s:%(fix_version_start_including)s:*:*:*:*:*:*:*","versionStartExcluding":"%(fix_version_start_excluding)s","versionEndExcluding":"%(fix_version_end_excluding)s","versionStartIncluding":"%(fix_version_start_including)s","versionEndIncluding":"%(fix_version_end_including)s"}]}]},"impact":{"baseMetricV3":{"cvssV3":{"version":"3.1","vectorString":"%(vectorString)s","attackVector":"NETWORK","attackComplexity":"%(attackComplexity)s","privilegesRequired":"NONE","userInteraction":"REQUIRED","scope":"UNCHANGED","confidentialityImpact":"%(severity)s","integrityImpact":"%(severity)s","availabilityImpact":"%(severity)s","baseScore":%(score).1f,"baseSeverity":"%(severity)s"},"exploitabilityScore":%(exploitabilityScore).1f,"impactScore":%(score).1f},"baseMetricV2":{"cvssV2":{"version":"2.0","vectorString":"AV:N/AC:M/Au:N/C:P/I:P/A:P","accessVector":"NETWORK","accessComplexity":"MEDIUM","authentication":"NONE","confidentialityImpact":"PARTIAL","integrityImpact":"PARTIAL","availabilityImpact":"PARTIAL","baseScore":%(score).1f},"severity":"%(severity)s","exploitabilityScore":%(exploitabilityScore).1f,"impactScore":%(score).1f,"acInsufInfo":false,"obtainAllPrivilege":false,"obtainUserPrivilege":false,"obtainOtherPrivilege":false,"userInteractionRequired":false}},"publishedDate":"%(publishedDate)s","lastModifiedDate":"%(lastModifiedDate)s"}
 """
 
+osv_url_dict = {
+    "javascript": "https://osv-vulnerabilities.storage.googleapis.com/npm/all.zip",
+    "python": "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip",
+    "go": "https://osv-vulnerabilities.storage.googleapis.com/Go/all.zip",
+    "java": "https://osv-vulnerabilities.storage.googleapis.com/Maven/all.zip",
+    "rust": "https://osv-vulnerabilities.storage.googleapis.com/crates.io/all.zip",
+    "csharp": "https://osv-vulnerabilities.storage.googleapis.com/NuGet/all.zip",
+    "ruby": "https://osv-vulnerabilities.storage.googleapis.com/RubyGems/all.zip",
+}
 # CVE types to exclude - hardware
 nvd_exclude_types = ["h"]
 if os.getenv("NVD_EXCLUDE_TYPES") is not None: