codeoceansdk.Pipeline

  1import logging
  2from dataclasses import asdict, dataclass, field as dc_field
  3from enum import Enum
  4from typing import Optional
  5
  6from codeoceansdk.Computation import Computation
  7from codeoceansdk.CodeOcean import CodeOcean
  8from codeoceansdk.Capsule import (
  9    SubmissionInfo,
 10    Version,
 11    Article,
 12    OriginalCapsuleInfo,
 13    UserPermission,
 14    GroupPermission,
 15    EveryonePermission,
 16)
 17
 18logger = logging.getLogger(__name__)
 19
 20
 21@dataclass(kw_only=True)
 22class Pipeline(CodeOcean):
 23    id: str
 24    """Pipeline internal id"""
 25    created: int = 0
 26    """Pipeline creation time"""
 27    name: str = ""
 28    """Pipeline display name"""
 29    status: Enum(
 30        "status", ["non_published", "submitted", "publishing", "published", "verified"]
 31    ) = "non_published"
 32    """Whether or not pipeline is published"""
 33    owner: str = ""
 34    """Pipeline owner id"""
 35    slug: str = ""
 36    """Alternate pipeline id"""
 37    original_capsule: Optional[OriginalCapsuleInfo] = None
 38    """Original capsule info (if duplicated) (optional)"""
 39    published_capsule: str = None
 40    """Published capsule id (separate from original)"""
 41    submission: Optional[SubmissionInfo] = None
 42    """Submission info (if pipeline submitted for publication) (optional)"""
 43    versions: Optional[list[Version]] = dc_field(default_factory=list)
 44    """Pipeline versions (if published) (optional)"""
 45    description: Optional[str] = None
 46    """Capsule description"""
 47    field: Optional[str] = None
 48    """Pipeline research field"""
 49    keywords: Optional[list[str]] = dc_field(default_factory=list)
 50    """Keywords describing pipeline (optional)"""
 51    article: Optional[Article] = None
 52    """Pipeline article info (optional)"""
 53    cloned_from_url: Optional[str] = None
 54    """If this is a pipeline cloned from github, what url was it"""
 55
 56    def __post_init__(self):
 57        super().__post_init__()
 58        self.pipeline_url = f"{self.api_url}/pipelines/{self.id}"
 59
 60    def _parse_comp_response(self, computation_list: list):
 61        """
 62        Parse response from list of dictionaries containing
 63        computational parameters
 64
 65        :param computation_list:  Input list of dictionary of computation parameters
 66        :return: list of Computation objects
 67        """
 68        computations = []
 69        for curr_dict in computation_list:
 70            computations.append(
 71                Computation.from_dict(
 72                    computation_dict=curr_dict, api_key=self.api_key, domain=self.domain
 73                )
 74            )
 75        return computations
 76
 77    @staticmethod
 78    def from_dict(dataset_dict, domain, api_key):
 79        """
 80
 81        :param dataset_dict: Dictionary containing Dataset parameters
 82        :param domain: Code Ocean Domain
 83        :param api_key: API key to access data asset
 84        :return: DataAsset
 85        """
 86        if "original_capsule" in dataset_dict:
 87            dataset_dict["original_capsule"] = OriginalCapsuleInfo(
 88                **dataset_dict["original_capsule"]
 89            )
 90        if "versions" in dataset_dict:
 91            dataset_dict["versions"] = [Version(**x) for x in dataset_dict["versions"]]
 92
 93        dataset_dict["domain"] = domain
 94        dataset_dict["api_key"] = api_key
 95        return Pipeline(**dataset_dict)
 96
 97    def get_pipeline(self):
 98        """
 99        Get pipeline metadata by ID
100
101        :return: None
102        """
103        req = self.get(self.pipeline_url)
104        new_pipeline = self.from_dict(req.json(), self.domain, self.api_key)
105        self.__dict__.update(new_pipeline.__dict__)
106
107    def get_pipeline_runs(self):
108        """Get previous capsule runs
109
110        :return: List of Computation objects.
111        """
112        input_url = f"{self.pipeline_url}/computations"
113        logger.debug(f"Input url: {input_url}")
114        req = self.get(input_url)
115        computations = self._parse_comp_response(computation_list=req.json())
116
117        logger.info("Returned runs: {}".format(len(computations)))
118        return computations
119
120    def set_capsule_permissions(
121        self,
122        users: list[UserPermission] = None,
123        groups: list[GroupPermission] = None,
124        everyone: EveryonePermission = None,
125    ):
126        """
127        Set capsule permissions
128        :param users: User permissions to set
129        :param groups: Group permissions to set
130        :param everyone: Permissions for everyone with access to the capsule
131        """
132        input_url = f"{self.api_url}/pipelines/{self.id}/permissions"
133
134        logger.debug(f"Input url: {input_url}")
135        logger.debug(f"Users {users}")
136        logger.debug(f"groups {groups}")
137        logger.debug(f"everyone {everyone}")
138
139        payload = {}
140        if users:
141            payload["users"] = [asdict(x) for x in users]
142        if groups:
143            payload["groups"] = [asdict(x) for x in groups]
144        if everyone:
145            payload["everyone"] = asdict(everyone)["role"]
146
147        self.post(input_url, payload)
148
149    def run_pipeline_computation(self, data_assets: list = None):
150        """
151        Run a capsule.
152
153        :param data_assets: List of dictionaries containing "id" and "mount" keys.
154        id is the data asset id, mount is the location to mount it in the capsule.
155        :return: Computation object.
156        """
157        logger.info(f"Running pipeline computation on {self.id}")
158        input_url = f"{self.api_url}/computations"
159
160        logger.debug(f"Input url: {input_url}")
161        logger.debug(f"Data assets {data_assets}")
162
163        payload = {"pipeline_id": self.id}
164
165        if data_assets:
166            payload["data_assets"] = data_assets
167        req = self.post(input_url, payload)
168        return Computation.from_dict(req.json(), self.domain, self.api_key)
logger = <Logger codeoceansdk.Pipeline (WARNING)>
@dataclass(kw_only=True)
class Pipeline(codeoceansdk.CodeOcean.CodeOcean):
 22@dataclass(kw_only=True)
 23class Pipeline(CodeOcean):
 24    id: str
 25    """Pipeline internal id"""
 26    created: int = 0
 27    """Pipeline creation time"""
 28    name: str = ""
 29    """Pipeline display name"""
 30    status: Enum(
 31        "status", ["non_published", "submitted", "publishing", "published", "verified"]
 32    ) = "non_published"
 33    """Whether or not pipeline is published"""
 34    owner: str = ""
 35    """Pipeline owner id"""
 36    slug: str = ""
 37    """Alternate pipeline id"""
 38    original_capsule: Optional[OriginalCapsuleInfo] = None
 39    """Original capsule info (if duplicated) (optional)"""
 40    published_capsule: str = None
 41    """Published capsule id (separate from original)"""
 42    submission: Optional[SubmissionInfo] = None
 43    """Submission info (if pipeline submitted for publication) (optional)"""
 44    versions: Optional[list[Version]] = dc_field(default_factory=list)
 45    """Pipeline versions (if published) (optional)"""
 46    description: Optional[str] = None
 47    """Capsule description"""
 48    field: Optional[str] = None
 49    """Pipeline research field"""
 50    keywords: Optional[list[str]] = dc_field(default_factory=list)
 51    """Keywords describing pipeline (optional)"""
 52    article: Optional[Article] = None
 53    """Pipeline article info (optional)"""
 54    cloned_from_url: Optional[str] = None
 55    """If this is a pipeline cloned from github, what url was it"""
 56
 57    def __post_init__(self):
 58        super().__post_init__()
 59        self.pipeline_url = f"{self.api_url}/pipelines/{self.id}"
 60
 61    def _parse_comp_response(self, computation_list: list):
 62        """
 63        Parse response from list of dictionaries containing
 64        computational parameters
 65
 66        :param computation_list:  Input list of dictionary of computation parameters
 67        :return: list of Computation objects
 68        """
 69        computations = []
 70        for curr_dict in computation_list:
 71            computations.append(
 72                Computation.from_dict(
 73                    computation_dict=curr_dict, api_key=self.api_key, domain=self.domain
 74                )
 75            )
 76        return computations
 77
 78    @staticmethod
 79    def from_dict(dataset_dict, domain, api_key):
 80        """
 81
 82        :param dataset_dict: Dictionary containing Dataset parameters
 83        :param domain: Code Ocean Domain
 84        :param api_key: API key to access data asset
 85        :return: DataAsset
 86        """
 87        if "original_capsule" in dataset_dict:
 88            dataset_dict["original_capsule"] = OriginalCapsuleInfo(
 89                **dataset_dict["original_capsule"]
 90            )
 91        if "versions" in dataset_dict:
 92            dataset_dict["versions"] = [Version(**x) for x in dataset_dict["versions"]]
 93
 94        dataset_dict["domain"] = domain
 95        dataset_dict["api_key"] = api_key
 96        return Pipeline(**dataset_dict)
 97
 98    def get_pipeline(self):
 99        """
100        Get pipeline metadata by ID
101
102        :return: None
103        """
104        req = self.get(self.pipeline_url)
105        new_pipeline = self.from_dict(req.json(), self.domain, self.api_key)
106        self.__dict__.update(new_pipeline.__dict__)
107
108    def get_pipeline_runs(self):
109        """Get previous capsule runs
110
111        :return: List of Computation objects.
112        """
113        input_url = f"{self.pipeline_url}/computations"
114        logger.debug(f"Input url: {input_url}")
115        req = self.get(input_url)
116        computations = self._parse_comp_response(computation_list=req.json())
117
118        logger.info("Returned runs: {}".format(len(computations)))
119        return computations
120
121    def set_capsule_permissions(
122        self,
123        users: list[UserPermission] = None,
124        groups: list[GroupPermission] = None,
125        everyone: EveryonePermission = None,
126    ):
127        """
128        Set capsule permissions
129        :param users: User permissions to set
130        :param groups: Group permissions to set
131        :param everyone: Permissions for everyone with access to the capsule
132        """
133        input_url = f"{self.api_url}/pipelines/{self.id}/permissions"
134
135        logger.debug(f"Input url: {input_url}")
136        logger.debug(f"Users {users}")
137        logger.debug(f"groups {groups}")
138        logger.debug(f"everyone {everyone}")
139
140        payload = {}
141        if users:
142            payload["users"] = [asdict(x) for x in users]
143        if groups:
144            payload["groups"] = [asdict(x) for x in groups]
145        if everyone:
146            payload["everyone"] = asdict(everyone)["role"]
147
148        self.post(input_url, payload)
149
150    def run_pipeline_computation(self, data_assets: list = None):
151        """
152        Run a capsule.
153
154        :param data_assets: List of dictionaries containing "id" and "mount" keys.
155        id is the data asset id, mount is the location to mount it in the capsule.
156        :return: Computation object.
157        """
158        logger.info(f"Running pipeline computation on {self.id}")
159        input_url = f"{self.api_url}/computations"
160
161        logger.debug(f"Input url: {input_url}")
162        logger.debug(f"Data assets {data_assets}")
163
164        payload = {"pipeline_id": self.id}
165
166        if data_assets:
167            payload["data_assets"] = data_assets
168        req = self.post(input_url, payload)
169        return Computation.from_dict(req.json(), self.domain, self.api_key)
Pipeline( *, domain: str, api_key: str, id: str, created: int = 0, name: str = '', status: codeoceansdk.Pipeline.status = 'non_published', owner: str = '', slug: str = '', original_capsule: Optional[codeoceansdk.Capsule.OriginalCapsuleInfo] = None, published_capsule: str = None, submission: Optional[codeoceansdk.Capsule.SubmissionInfo] = None, versions: Optional[list[codeoceansdk.Capsule.Version]] = <factory>, description: Optional[str] = None, field: Optional[str] = None, keywords: Optional[list[str]] = <factory>, article: Optional[codeoceansdk.Capsule.Article] = None, cloned_from_url: Optional[str] = None)
id: str

Pipeline internal id

created: int = 0

Pipeline creation time

name: str = ''

Pipeline display name

status: codeoceansdk.Pipeline.status = 'non_published'

Whether or not pipeline is published

owner: str = ''

Pipeline owner id

slug: str = ''

Alternate pipeline id

original_capsule: Optional[codeoceansdk.Capsule.OriginalCapsuleInfo] = None

Original capsule info (if duplicated) (optional)

published_capsule: str = None

Published capsule id (separate from original)

submission: Optional[codeoceansdk.Capsule.SubmissionInfo] = None

Submission info (if pipeline submitted for publication) (optional)

versions: Optional[list[codeoceansdk.Capsule.Version]]

Pipeline versions (if published) (optional)

description: Optional[str] = None

Capsule description

field: Optional[str] = None

Pipeline research field

keywords: Optional[list[str]]

Keywords describing pipeline (optional)

article: Optional[codeoceansdk.Capsule.Article] = None

Pipeline article info (optional)

cloned_from_url: Optional[str] = None

If this is a pipeline cloned from github, what url was it

@staticmethod
def from_dict(dataset_dict, domain, api_key):
78    @staticmethod
79    def from_dict(dataset_dict, domain, api_key):
80        """
81
82        :param dataset_dict: Dictionary containing Dataset parameters
83        :param domain: Code Ocean Domain
84        :param api_key: API key to access data asset
85        :return: DataAsset
86        """
87        if "original_capsule" in dataset_dict:
88            dataset_dict["original_capsule"] = OriginalCapsuleInfo(
89                **dataset_dict["original_capsule"]
90            )
91        if "versions" in dataset_dict:
92            dataset_dict["versions"] = [Version(**x) for x in dataset_dict["versions"]]
93
94        dataset_dict["domain"] = domain
95        dataset_dict["api_key"] = api_key
96        return Pipeline(**dataset_dict)
Parameters
  • dataset_dict: Dictionary containing Dataset parameters
  • domain: Code Ocean Domain
  • api_key: API key to access data asset
Returns

DataAsset

def get_pipeline(self):
 98    def get_pipeline(self):
 99        """
100        Get pipeline metadata by ID
101
102        :return: None
103        """
104        req = self.get(self.pipeline_url)
105        new_pipeline = self.from_dict(req.json(), self.domain, self.api_key)
106        self.__dict__.update(new_pipeline.__dict__)

Get pipeline metadata by ID

Returns

None

def get_pipeline_runs(self):
108    def get_pipeline_runs(self):
109        """Get previous capsule runs
110
111        :return: List of Computation objects.
112        """
113        input_url = f"{self.pipeline_url}/computations"
114        logger.debug(f"Input url: {input_url}")
115        req = self.get(input_url)
116        computations = self._parse_comp_response(computation_list=req.json())
117
118        logger.info("Returned runs: {}".format(len(computations)))
119        return computations

Get previous capsule runs

Returns

List of Computation objects.

def set_capsule_permissions( self, users: list[codeoceansdk.Capsule.UserPermission] = None, groups: list[codeoceansdk.Capsule.GroupPermission] = None, everyone: codeoceansdk.Capsule.EveryonePermission = None):
121    def set_capsule_permissions(
122        self,
123        users: list[UserPermission] = None,
124        groups: list[GroupPermission] = None,
125        everyone: EveryonePermission = None,
126    ):
127        """
128        Set capsule permissions
129        :param users: User permissions to set
130        :param groups: Group permissions to set
131        :param everyone: Permissions for everyone with access to the capsule
132        """
133        input_url = f"{self.api_url}/pipelines/{self.id}/permissions"
134
135        logger.debug(f"Input url: {input_url}")
136        logger.debug(f"Users {users}")
137        logger.debug(f"groups {groups}")
138        logger.debug(f"everyone {everyone}")
139
140        payload = {}
141        if users:
142            payload["users"] = [asdict(x) for x in users]
143        if groups:
144            payload["groups"] = [asdict(x) for x in groups]
145        if everyone:
146            payload["everyone"] = asdict(everyone)["role"]
147
148        self.post(input_url, payload)

Set capsule permissions

Parameters
  • users: User permissions to set
  • groups: Group permissions to set
  • everyone: Permissions for everyone with access to the capsule
def run_pipeline_computation(self, data_assets: list = None):
150    def run_pipeline_computation(self, data_assets: list = None):
151        """
152        Run a capsule.
153
154        :param data_assets: List of dictionaries containing "id" and "mount" keys.
155        id is the data asset id, mount is the location to mount it in the capsule.
156        :return: Computation object.
157        """
158        logger.info(f"Running pipeline computation on {self.id}")
159        input_url = f"{self.api_url}/computations"
160
161        logger.debug(f"Input url: {input_url}")
162        logger.debug(f"Data assets {data_assets}")
163
164        payload = {"pipeline_id": self.id}
165
166        if data_assets:
167            payload["data_assets"] = data_assets
168        req = self.post(input_url, payload)
169        return Computation.from_dict(req.json(), self.domain, self.api_key)

Run a capsule.

Parameters
  • data_assets: List of dictionaries containing "id" and "mount" keys. id is the data asset id, mount is the location to mount it in the capsule.
Returns

Computation object.