Skip to content

Instantly share code, notes, and snippets.

@jmaher

jmaher/.taskcluster.yml

Forked from grenade/.taskcluster.yml
Last active July 8, 2020 14:17
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jmaher/1387efecbfd9f53d2bf26d6674e25805 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jmaher/1387efecbfd9f53d2bf26d6674e25805 to your computer and use it in GitHub Desktop.
Download ZIP
repeat a set of tasks (a configurable number of times) each with a set of commands on multiple taskcluster worker types
Raw
.taskcluster.yml
---
provisionerId: relops-3
workerType: decision
retries: 5
priority: highest
created: '2020-07-08T13:00:36.378Z'
deadline: '2020-07-08T16:00:36.378Z'
routes:
- index.project.relops.worker-details.decision
scopes:
- 'queue:route:index.project.relops.worker-details.*'
- 'queue:scheduler-id:-'
- 'queue:create-task:low:gecko-t/t-win*'
- 'queue:task-group-id:taskcluster-github/*'
- 'queue:schedule-task:taskcluster-github/*'
- 'queue:create-task:highest:relops-3/decision'
payload:
image: python
maxRunTime: 600
features:
taskclusterProxy: true
artifacts:
public/results.json:
path: results.json
type: file
command:
- /bin/bash
- '--login'
- '-c'
- >-
python --version
&& pip install asyncio | grep -v "^[[:space:]]*$"
&& pip install pyyaml | grep -v "^[[:space:]]*$"
&& pip install slugid | grep -v "^[[:space:]]*$"
&& pip install taskcluster | grep -v "^[[:space:]]*$"
&& git clone --quiet https://gist.github.com/1387efecbfd9f53d2bf26d6674e25805.git ./worker-observation
&& python ./worker-observation/decision-task.py
metadata:
owner: [email protected]
source: 'https://gist.github.com/jmaher/1387efecbfd9f53d2bf26d6674e25805'
name: '00 :: create-worker-tasks'
description: create observation tasks on workers
tags: {}
extra: {}
Raw
config.yml
---
tasks:
- targets:
- provisioner: releng-hardware
workertype: t-win10-64-hw
iterations: 1
- provisioner: gecko-t
workertype: t-win10-64
iterations: 1
- provisioner: gecko-t
workertype: t-win10-64
iterations: 1
- provisioner: gecko-t
workertype: t-win10-64-gpu-s
iterations: 1
- provisioner: gecko-t
workertype: t-win7-32
iterations: 1
- provisioner: gecko-t
workertype: t-win7-32-gpu
iterations: 1
namespace: observe-software-versions
name:
prefix: '01 :: '
suffix: " :: observe-software-versions"
description:
prefix: 'observe software versions on '
suffix: ''
owner: [email protected]
command:
- mkdir .\public
- chcp 65001 > nul && ver > .\public\windows-version.txt
- chcp 65001 > nul && C:\mozilla-build\python\python.exe --version 2> .\public\python2-version.txt
- chcp 65001 > nul && C:\mozilla-build\python3\python3.exe --version > .\public\python3-version.txt
- >-
chcp 65001 > nul && C:\mozilla-build\python\python.exe -c "exec(\"import
platform;print platform.release()\")" >
.\public\python2-platform-release.txt
- >-
chcp 65001 > nul && C:\mozilla-build\python3\python3.exe -c "exec(\"import
platform;print(platform.release())\")" >
.\public\python3-platform-release.txt
- chcp 65001 > nul && wget.exe --version > .\public\wget-version.txt
- chcp 65001 > nul && hg --version > .\public\hg-version.txt
- chcp 65001 > nul && C:\generic-worker\generic-worker.exe --version > .\public\generic-worker-version.txt
- chcp 65001 > nul && wmic os get servicepackmajorversion > .\public\servicepack-major-version.txt
- chcp 65001 > nul && "C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi" > .\public\nvidia-details.txt
artifacts:
- type: file
name: public/windows-version.txt
path: public/windows-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python2-version.txt
path: public/python2-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python3-version.txt
path: public/python3-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python2-platform-release.txt
path: public/python2-platform-release.txt
- type: file
name: public/python3-platform-release.txt
path: public/python3-platform-release.txt
- type: file
name: public/wget-version.txt
path: public/wget-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/hg-version.txt
path: public/hg-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/generic-worker-version.txt
path: public/generic-worker-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/nvidia-details.txt
path: public/nvidia-details.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/servicepack-major-version.txt
path: public/servicepack-major-version.txt
encoding: utf-16
line: 1
features:
taskclusterProxy: true
maxruntime: 600
Raw
debug-config.yml
---
tasks:
- targets:
- provisioner: aws-provisioner-v1
workertype: gecko-t-win7-32-gpu
iterations: 1
- provisioner: releng-hardware
workertype: gecko-t-win10-64-hw
iterations: 1
namespace: collect-software-versions
name:
prefix: '01 :: '
suffix: " :: collect-software-versions"
description:
prefix: 'retrieve software versions on '
suffix: ''
owner: [email protected]
command:
- mkdir .\public
- chcp 65001 > nul && ver > .\public\windows-version.txt
- chcp 65001 > nul && C:\mozilla-build\python\python.exe --version 2> .\public\python2-version.txt
- chcp 65001 > nul && C:\mozilla-build\python3\python3.exe --version > .\public\python3-version.txt
- >-
chcp 65001 > nul && C:\mozilla-build\python\python.exe -c "exec(\"import
platform;print platform.release()\")" >
.\public\python2-platform-release.txt
- >-
chcp 65001 > nul && C:\mozilla-build\python3\python3.exe -c "exec(\"import
platform;print(platform.release())\")" >
.\public\python3-platform-release.txt
- chcp 65001 > nul && wget.exe --version > .\public\wget-version.txt
- chcp 65001 > nul && hg --version > .\public\hg-version.txt
- chcp 65001 > nul && C:\generic-worker\generic-worker.exe --version > .\public\generic-worker-version.txt
- chcp 65001 > nul && wmic os get servicepackmajorversion > .\public\servicepack-major-version.txt
artifacts:
- type: file
name: public/windows-version.txt
path: public/windows-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python2-version.txt
path: public/python2-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python3-version.txt
path: public/python3-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/python2-platform-release.txt
path: public/python2-platform-release.txt
- type: file
name: public/python3-platform-release.txt
path: public/python3-platform-release.txt
- type: file
name: public/wget-version.txt
path: public/wget-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/hg-version.txt
path: public/hg-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/generic-worker-version.txt
path: public/generic-worker-version.txt
regex:
match: '\d+\.\d+\.\d+'
group: 0
- type: file
name: public/servicepack-major-version.txt
path: public/servicepack-major-version.txt
encoding: utf-16
line: 1
features:
taskclusterProxy: true
maxruntime: 600
- targets:
- provisioner: aws-provisioner-v1
workertype: gecko-t-win7-32-gpu
iterations: 1
namespace: collect-occ-config
name:
prefix: '02 :: '
suffix: " :: collect-occ-config"
description:
prefix: 'determine occ configuration on '
suffix: ''
owner: [email protected]
command:
- mkdir .\public
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Organisation > .\public\occ-source-organisation.txt
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Repository > .\public\occ-source-repository.txt
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\OpenCloudConfig\Source /v Revision > .\public\occ-source-revision.txt
artifacts:
- type: file
name: public/occ-source-organisation.txt
path: public/occ-source-organisation.txt
line: 2
split:
separator: " "
index: 2
- type: file
name: public/occ-source-repository.txt
path: public/occ-source-repository.txt
line: 2
split:
separator: " "
index: 2
- type: file
name: public/occ-source-revision.txt
path: public/occ-source-revision.txt
line: 2
split:
separator: " "
index: 2
features:
taskclusterProxy: true
maxruntime: 600
- targets:
- provisioner: releng-hardware
workertype: gecko-t-win10-64-hw
iterations: 1
namespace: collect-ronin-config
name:
prefix: '03 :: '
suffix: " :: collect-ronin-config"
description:
prefix: 'determine ronin configuration on '
suffix: ''
owner: [email protected]
command:
- mkdir .\public
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Organisation > .\public\ronin-source-organisation.txt
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Repository > .\public\ronin-source-repository.txt
- chcp 65001 > nul && reg query HKLM\SOFTWARE\Mozilla\ronin_puppet\Source /v Revision > .\public\ronin-source-revision.txt
artifacts:
- type: file
name: public/ronin-source-organisation.txt
path: public/ronin-source-organisation.txt
line: 2
split:
separator: " "
index: 2
- type: file
name: public/ronin-source-repository.txt
path: public/ronin-source-repository.txt
line: 2
split:
separator: " "
index: 2
- type: file
name: public/ronin-source-revision.txt
path: public/ronin-source-revision.txt
line: 2
split:
separator: " "
index: 2
features:
taskclusterProxy: true
maxruntime: 600
Raw
decision-task.py
import asyncio
import os
import json
import re
import slugid
import taskcluster.aio
import time
import urllib
import urllib.request
import yaml
from datetime import datetime, timedelta
from gzip import decompress
from random import randint, seed
seed(1)
GIST_USER = 'jmaher'
GIST_SHA = '1387efecbfd9f53d2bf26d6674e25805'
async def create_task(provisioner, workerType, taskGroupId, task, iteration, iterations):
global asyncQueue
taskId = slugid.nice()
payload = {
'created': '{}Z'.format(datetime.utcnow().isoformat()[:-3]),
'deadline': '{}Z'.format((datetime.utcnow() + timedelta(days=3)).isoformat()[:-3]),
'provisionerId': provisioner,
'workerType': workerType,
'taskGroupId': taskGroupId,
'routes': [
'index.project.relops.worker-details.{}.{}.{}'.format(provisioner, workerType, task['namespace']),
'index.project.relops.worker-details.latest.{}.{}.{}'.format(provisioner, workerType, task['namespace']),
'index.project.relops.worker-details.daily.{}.{}.{}.{}'.format(datetime.utcnow().strftime("%Y%m%d"), provisioner, workerType, task['namespace']),
'index.project.relops.worker-details.hourly.{}.{}.{}.{}'.format(datetime.utcnow().strftime("%Y%m%d%H"), provisioner, workerType, task['namespace'])
],
'scopes': [],
'payload': {
'maxRunTime': task['maxruntime'],
'command': task['command'],
'artifacts': list(map(lambda x: {'type': x['type'],'name': x['name'],'path': x['path']}, task['artifacts'])),
'features': task['features']
},
'metadata': {
'name': '{} {}/{} {} :: {}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations) if (iterations < 10) else '{}{}/{}{} {:02d}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations) if (iterations < 100) else '{}{}/{}{} {:03d}/{}'.format(task['name']['prefix'], provisioner, workerType, task['name']['suffix'], iteration, iterations),
'description': '{}{}/{}{}'.format(task['description']['prefix'], provisioner, workerType, task['description']['suffix']),
'owner': task['owner'],
'source': 'https://gist.github.com/{}/{}'.format(GIST_USER, GIST_SHA)
}
}
print('creating {}/{} task {} ({}/{} {}) ({}/groups/{}/tasks/{})'.format(provisioner, workerType, task['namespace'], iteration, iterations, taskId, os.environ.get('TASKCLUSTER_ROOT_URL'), os.environ.get('TASK_ID'), taskId))
return await asyncQueue.createTask(taskId, payload)
async def print_task_artifacts(provisioner, workerType, taskGroupId, taskNamespace, task, iteration, iterations):
global results, asyncQueue
taskStatus = await create_task(provisioner, workerType, taskGroupId, task, iteration, iterations)
print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state']))
while taskStatus['status']['state'] not in ['completed', 'failed']:
await asyncio.sleep(randint(10, 30) if taskStatus['status']['state'] == 'pending' else 0.5)
print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state']))
taskStatus = await asyncQueue.status(taskStatus['status']['taskId'])
print('{}/{} - {} ({}/{} {}): {} on run {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], taskStatus['status']['state'], taskStatus['status']['runs'][-1]['runId']))
for artifactDefinition in task['artifacts']:
artifactUrl = 'https://taskcluster-artifacts.net/{}/{}/{}'.format(taskStatus['status']['taskId'], taskStatus['status']['runs'][-1]['runId'], artifactDefinition['name'])
print('{}/{} - {} ({}/{} {}): {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], artifactUrl))
try:
artifactContent = decompress(urllib.request.urlopen(urllib.request.Request(artifactUrl)).read()).decode(artifactDefinition['encoding'] if 'encoding' in artifactDefinition else 'utf-8')
if 'file-missing-on-worker' in artifactContent:
artifactContent = None
except Exception as e:
print('error fetching artifact {}'.format(artifactUrl), e)
artifactContent = None
if artifactContent is not None:
artifactLine = artifactContent.split('\n')[artifactDefinition['line']].strip() if 'line' in artifactDefinition else artifactContent.strip()
artifactText = artifactLine.split(artifactDefinition['split']['separator'])[artifactDefinition['split']['index']].strip() if 'split' in artifactDefinition else artifactLine
if 'regex' in artifactDefinition:
try:
artifactText = re.search(artifactDefinition['regex']['match'], artifactText).group(artifactDefinition['regex']['group'])
except Exception as e:
artifactText = ''
print('error matching regex: "{}", group: {}'.format(artifactDefinition['regex']['match'], artifactDefinition['regex']['group']), e)
else:
artifactText = ''
print('{}/{} - {} ({}/{} {}): {}: {}'.format(provisioner, workerType, taskNamespace, iteration, iterations, taskStatus['status']['taskId'], artifactDefinition['name'], artifactText))
run = taskStatus['status']['runs'][-1]['runId']
if workerType in results:
if taskNamespace in results[workerType]:
if 'iteration-{}'.format(iteration) in results[workerType][taskNamespace]:
results[workerType][taskNamespace]['iteration-{}'.format(iteration)].update({ 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText })
else:
results[workerType][taskNamespace].update({ 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } })
else:
results[workerType].update({ taskNamespace: { 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } } })
else:
results.update({ workerType: { taskNamespace: { 'iteration-{}'.format(iteration): { 'task': taskStatus['status'], os.path.splitext(os.path.basename(artifactDefinition['name']))[run]: artifactText } } } })
async def close(session):
await session.close()
config = yaml.load(urllib.request.urlopen('https://gist.githubusercontent.com/{}/{}/raw/config.yml?{}'.format(GIST_USER, GIST_SHA, slugid.nice())).read())
taskclusterOptions = {
'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']
}
start = time.time()
loop = asyncio.get_event_loop()
session = taskcluster.aio.createSession(loop=loop)
asyncQueue = taskcluster.aio.Queue(taskclusterOptions, session=session)
tasks = []
results = {}
for task in config['tasks']:
for target in task['targets']:
for i in range(1, (target['iterations'] + 1)):
tasks.append(asyncio.ensure_future(print_task_artifacts(target['provisioner'], target['workertype'], os.environ.get('TASK_ID'), task['namespace'], task, i, target['iterations'])))
loop.run_until_complete(asyncio.wait(tasks, timeout=1200))
loop.run_until_complete(close(session))
loop.close()
with open('results.json', 'w') as fp:
json.dump(results, fp, indent=2, sort_keys=True)
end = time.time()
print("total time: {}".format(end - start))
Raw
get-chrome-version.cmd
@ECHO OFF
:: Look for machine-wide Chrome installs (stable, Beta, and Dev).
:: Get the name, running version (if an update is pending relaunch), and
:: installed version of each.
FOR %%A IN (
{8A69D345-D564-463c-AFF1-A69D9E530F96},
{8237E44A-0054-442C-B6B6-EA0509993955},
{401C381F-E0DE-4B85-8BD8-3F3F14FBDA57},
{4ea16ac7-fd5a-47c3-875b-dbf4a2008c20}) DO (
reg query HKLM\Software\Google\Update\Clients\%%A > NUL 2>&1
if %errorlevel% equ 0 (
reg query HKLM\Software\Google\Update\Clients\%%A /v name > NUL 2>&1
if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v name /reg:32 2> NUL
reg query HKLM\Software\Google\Update\Clients\%%A /v opv > NUL 2>&1
if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v opv /reg:32 2> NUL
reg query HKLM\Software\Google\Update\Clients\%%A /v pv > NUL 2>&1
if %errorlevel% equ 0 reg query HKLM\Software\Google\Update\Clients\%%A /v pv /reg:32 2> NUL
)
)
:: Look for Chrome installs in the current user's %LOCALAPPDATA% directory
:: (stable, Beta, Dev, and canary).
:: Get the name, running version (if an update is pending relaunch), and
:: installed version of each.
FOR %%A IN (
{8A69D345-D564-463c-AFF1-A69D9E530F96},
{8237E44A-0054-442C-B6B6-EA0509993955},
{401C381F-E0DE-4B85-8BD8-3F3F14FBDA57},
{4ea16ac7-fd5a-47c3-875b-dbf4a2008c20}) DO (
reg query HKCU\Software\Google\Update\Clients\%%A > NUL 2>&1
if %errorlevel% equ 0 (
reg query HKCU\Software\Google\Update\Clients\%%A /v name > NUL 2>&1
if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v name /reg:32 2> NUL
reg query HKCU\Software\Google\Update\Clients\%%A /v opv > NUL 2>&1
if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v opv /reg:32 2> NUL
reg query HKCU\Software\Google\Update\Clients\%%A /v pv > NUL 2>&1
if %errorlevel% equ 0 reg query HKCU\Software\Google\Update\Clients\%%A /v pv /reg:32 2> NUL
)
)
EXIT 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment