d3mdm.d3m module

Classes

BaseManager([source, username, password, …])

BaseScraper([skip_sublevels])

D3MManager(username, password[, skip_sublevels])

D3MScraper(username, password[, skip_sublevels])

IPFSManager([skip_sublevels])

IPFSScraper([skip_sublevels])

class d3mdm.d3m.BaseManager(source='D3M', username=None, password=None, skip_sublevels=False)[source]

Bases: object

Methods

datasets()

exists(dataset_name)

load(dataset_name[, raw])

datasets()[source]
exists(dataset_name)[source]
load(dataset_name, raw='to_be_ignored')[source]
class d3mdm.d3m.BaseScraper(skip_sublevels=False)[source]

Bases: object

Attributes

BACKOFF_FACTOR

float(x) -> floating point number

DATASET_TYPES

dict() -> new empty dictionary

PATH

dict() -> new empty dictionary

RETRIES

int(x=0) -> integer

STATUS_FORCELIST

tuple() -> empty tuple

Methods

get_base_url(dataset_name)

get_datasets(dataset_type)

get_links(soup)

get_session()

get_sublevel(base_path, level, sublevels)

get_url(url[, raw])

scrape_dataset(dataset)

BACKOFF_FACTOR = 0.5
DATASET_TYPES = {}
PATH = {'ll0': 'training_datasets/LL0/', 'll1': 'training_datasets/LL1/', 'seed': 'seed_datasets_current/'}
RETRIES = 10
STATUS_FORCELIST = (403, 404, 500, 502, 504)
get_base_url(dataset_name)[source]
get_datasets(dataset_type)[source]
get_session()[source]
get_sublevel(base_path, level, sublevels)[source]
get_url(url, raw=False)[source]
scrape_dataset(dataset)[source]
class d3mdm.d3m.D3MManager(username, password, skip_sublevels=False)[source]

Bases: d3mdm.d3m.BaseManager

class d3mdm.d3m.D3MScraper(username, password, skip_sublevels=False)[source]

Bases: d3mdm.d3m.BaseScraper

Attributes

BASE_URL

str(object=’’) -> str

Methods

get_session(username, password)

BASE_URL = 'https://datadrivendiscovery.org/data/'
get_session(username, password)[source]
class d3mdm.d3m.IPFSManager(skip_sublevels=False)[source]

Bases: d3mdm.d3m.BaseManager

class d3mdm.d3m.IPFSScraper(skip_sublevels=False)[source]

Bases: d3mdm.d3m.BaseScraper

Attributes

BASE_URL

str(object=’’) -> str
BASE_URL = 'https://gateway.ipfs.io/ipfs/QmWsbzjogZTY3Laf8SErQ9azfuY7BWicBmQjP9SxwvtqTz/'