Add Frey support

Squashed commit of the following:

commit 9071377570a97d599f7fc769f4d26b45d03020cb
Author: Kevin van Zonneveld <kevin@vanzonneveld.net>
Date:   Tue Mar 29 13:36:57 2016 +0200

    Remove useless env code

commit 529aed3535d0159dd7291532ebb6cfcc9ae401c6
Author: Kevin van Zonneveld <kevin@vanzonneveld.net>
Date:   Tue Mar 29 13:31:38 2016 +0200

    Update state, add launch target

commit fde533f9f6cc896a7e2abd7e68320d806d90ba27
Author: Kevin van Zonneveld <kevin@vanzonneveld.net>
Date:   Tue Mar 29 11:07:35 2016 +0200

    First swing at moving Frey to tusd
This commit is contained in:
Kevin van Zonneveld 2016-03-29 13:37:19 +02:00
parent 199487bdf6
commit e54850a3aa
12 changed files with 468 additions and 1 deletions

5
.gitignore vendored
View File

@ -1,3 +1,8 @@
tusd/data tusd/data
cover.out cover.out
data/ data/
.infra/env.sh
.infra/env.infra.sh
.infra/ssh/infra-tusd.pem
.infra/Frey-residu*
.infra/ssh/frey-infra-tusd.pem

View File

@ -0,0 +1,119 @@
{
"version": 1,
"serial": 20,
"modules": [
{
"path": [
"root"
],
"outputs": {
"endpoint": "http://master.tus.io:80/",
"public_address": "ec2-54-161-247-36.compute-1.amazonaws.com",
"public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com"
},
"resources": {
"aws_instance.infra-tusd-server": {
"type": "aws_instance",
"primary": {
"id": "i-d59cff7e",
"attributes": {
"ami": "ami-9bce7af0",
"availability_zone": "us-east-1e",
"ebs_block_device.#": "0",
"ebs_optimized": "false",
"ephemeral_block_device.#": "0",
"iam_instance_profile": "",
"id": "i-d59cff7e",
"instance_state": "running",
"instance_type": "c3.large",
"key_name": "infra-tusd",
"monitoring": "false",
"private_dns": "ip-10-167-178-105.ec2.internal",
"private_ip": "10.167.178.105",
"public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com",
"public_ip": "54.161.247.36",
"root_block_device.#": "1",
"root_block_device.0.delete_on_termination": "true",
"root_block_device.0.iops": "24",
"root_block_device.0.volume_size": "8",
"root_block_device.0.volume_type": "gp2",
"security_groups.#": "1",
"security_groups.1246499019": "fw-infra-tusd-main",
"source_dest_check": "true",
"subnet_id": "",
"tags.#": "1",
"tags.Name": "master.tus.io",
"tenancy": "default",
"vpc_security_group_ids.#": "0"
},
"meta": {
"schema_version": "1"
}
}
},
"aws_route53_record.www": {
"type": "aws_route53_record",
"depends_on": [
"aws_instance.infra-tusd-server"
],
"primary": {
"id": "Z3IT8X6U91XY1P_master.tus.io_CNAME",
"attributes": {
"failover": "",
"fqdn": "master.tus.io",
"health_check_id": "",
"id": "Z3IT8X6U91XY1P_master.tus.io_CNAME",
"name": "master.tus.io",
"records.#": "1",
"records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com",
"set_identifier": "",
"ttl": "300",
"type": "CNAME",
"weight": "-1",
"zone_id": "Z3IT8X6U91XY1P"
}
}
},
"aws_security_group.fw-infra-tusd-main": {
"type": "aws_security_group",
"primary": {
"id": "sg-2ff78c42",
"attributes": {
"description": "Infra tusd",
"egress.#": "0",
"id": "sg-2ff78c42",
"ingress.#": "3",
"ingress.2214680975.cidr_blocks.#": "1",
"ingress.2214680975.cidr_blocks.0": "0.0.0.0/0",
"ingress.2214680975.from_port": "80",
"ingress.2214680975.protocol": "tcp",
"ingress.2214680975.security_groups.#": "0",
"ingress.2214680975.self": "false",
"ingress.2214680975.to_port": "80",
"ingress.516175195.cidr_blocks.#": "1",
"ingress.516175195.cidr_blocks.0": "0.0.0.0/0",
"ingress.516175195.from_port": "8080",
"ingress.516175195.protocol": "tcp",
"ingress.516175195.security_groups.#": "0",
"ingress.516175195.self": "false",
"ingress.516175195.to_port": "8080",
"ingress.614077637.cidr_blocks.#": "3",
"ingress.614077637.cidr_blocks.0": "62.163.187.106/32",
"ingress.614077637.cidr_blocks.1": "84.146.0.0/16",
"ingress.614077637.cidr_blocks.2": "24.134.75.132/32",
"ingress.614077637.from_port": "22",
"ingress.614077637.protocol": "tcp",
"ingress.614077637.security_groups.#": "0",
"ingress.614077637.self": "false",
"ingress.614077637.to_port": "22",
"name": "fw-infra-tusd-main",
"owner_id": "402421253186",
"tags.#": "0",
"vpc_id": ""
}
}
}
}
}
]
}

View File

@ -0,0 +1,119 @@
{
"version": 1,
"serial": 20,
"modules": [
{
"path": [
"root"
],
"outputs": {
"endpoint": "http://master.tus.io:80/",
"public_address": "ec2-54-161-247-36.compute-1.amazonaws.com",
"public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com"
},
"resources": {
"aws_instance.infra-tusd-server": {
"type": "aws_instance",
"primary": {
"id": "i-d59cff7e",
"attributes": {
"ami": "ami-9bce7af0",
"availability_zone": "us-east-1e",
"ebs_block_device.#": "0",
"ebs_optimized": "false",
"ephemeral_block_device.#": "0",
"iam_instance_profile": "",
"id": "i-d59cff7e",
"instance_state": "running",
"instance_type": "c3.large",
"key_name": "infra-tusd",
"monitoring": "false",
"private_dns": "ip-10-167-178-105.ec2.internal",
"private_ip": "10.167.178.105",
"public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com",
"public_ip": "54.161.247.36",
"root_block_device.#": "1",
"root_block_device.0.delete_on_termination": "true",
"root_block_device.0.iops": "24",
"root_block_device.0.volume_size": "8",
"root_block_device.0.volume_type": "gp2",
"security_groups.#": "1",
"security_groups.1246499019": "fw-infra-tusd-main",
"source_dest_check": "true",
"subnet_id": "",
"tags.#": "1",
"tags.Name": "master.tus.io",
"tenancy": "default",
"vpc_security_group_ids.#": "0"
},
"meta": {
"schema_version": "1"
}
}
},
"aws_route53_record.www": {
"type": "aws_route53_record",
"depends_on": [
"aws_instance.infra-tusd-server"
],
"primary": {
"id": "Z3IT8X6U91XY1P_master.tus.io_CNAME",
"attributes": {
"failover": "",
"fqdn": "master.tus.io",
"health_check_id": "",
"id": "Z3IT8X6U91XY1P_master.tus.io_CNAME",
"name": "master.tus.io",
"records.#": "1",
"records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com",
"set_identifier": "",
"ttl": "300",
"type": "CNAME",
"weight": "-1",
"zone_id": "Z3IT8X6U91XY1P"
}
}
},
"aws_security_group.fw-infra-tusd-main": {
"type": "aws_security_group",
"primary": {
"id": "sg-2ff78c42",
"attributes": {
"description": "Infra tusd",
"egress.#": "0",
"id": "sg-2ff78c42",
"ingress.#": "3",
"ingress.2214680975.cidr_blocks.#": "1",
"ingress.2214680975.cidr_blocks.0": "0.0.0.0/0",
"ingress.2214680975.from_port": "80",
"ingress.2214680975.protocol": "tcp",
"ingress.2214680975.security_groups.#": "0",
"ingress.2214680975.self": "false",
"ingress.2214680975.to_port": "80",
"ingress.516175195.cidr_blocks.#": "1",
"ingress.516175195.cidr_blocks.0": "0.0.0.0/0",
"ingress.516175195.from_port": "8080",
"ingress.516175195.protocol": "tcp",
"ingress.516175195.security_groups.#": "0",
"ingress.516175195.self": "false",
"ingress.516175195.to_port": "8080",
"ingress.614077637.cidr_blocks.#": "3",
"ingress.614077637.cidr_blocks.0": "62.163.187.106/32",
"ingress.614077637.cidr_blocks.1": "84.146.0.0/16",
"ingress.614077637.cidr_blocks.2": "24.134.75.132/32",
"ingress.614077637.from_port": "22",
"ingress.614077637.protocol": "tcp",
"ingress.614077637.security_groups.#": "0",
"ingress.614077637.self": "false",
"ingress.614077637.to_port": "22",
"name": "fw-infra-tusd-main",
"owner_id": "402421253186",
"tags.#": "0",
"vpc_id": ""
}
}
}
}
}
]
}

159
.infra/Freyfile.toml Normal file
View File

@ -0,0 +1,159 @@
[global]
appname = "infra-tusd"
[global.ssh]
key_dir = "./ssh"
[global.ansiblecfg.privilege_escalation]
become = true
[global.ansiblecfg.defaults]
host_key_checking = "False"
[infra.provider.aws]
access_key = "${var.FREY_AWS_ACCESS_KEY}"
region = "us-east-1"
secret_key = "${var.FREY_AWS_SECRET_KEY}"
[infra.variable.ami.default]
us-east-1 = "ami-9bce7af0"
[infra.variable.ip_all]
default = "0.0.0.0/0"
[infra.variable.ip_kevin]
default = "62.163.187.106/32"
[infra.variable.ip_marius]
default = "84.146.0.0/16"
[infra.variable.ip_tim]
default = "24.134.75.132/32"
[infra.variable.region]
default = "us-east-1"
[infra.output.public_address]
value = "${aws_instance.infra-tusd-server.0.public_dns}"
[infra.output.public_addresses]
value = "${join(\"\n\", aws_instance.infra-tusd-server.*.public_dns)}"
[infra.output.endpoint]
value = "http://${aws_route53_record.www.name}:80/"
[infra.resource.aws_instance.infra-tusd-server]
ami = "${lookup(var.ami, var.region)}"
instance_type = "c3.large"
key_name = "infra-tusd"
security_groups = ["fw-infra-tusd-main"]
[infra.resource.aws_instance.infra-tusd-server.connection]
key_file = "{{{config.global.ssh.privatekey_file}}}"
user = "{{{config.global.ssh.user}}}"
[infra.resource.aws_instance.infra-tusd-server.tags]
Name = "${var.FREY_DOMAIN}"
[infra.resource.aws_route53_record.www]
name = "${var.FREY_DOMAIN}"
records = ["${aws_instance.infra-tusd-server.public_dns}"]
ttl = "300"
type = "CNAME"
zone_id = "${var.FREY_AWS_ZONE_ID}"
[infra.resource.aws_security_group.fw-infra-tusd-main]
description = "Infra tusd"
name = "fw-infra-tusd-main"
[[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]]
cidr_blocks = ["${var.ip_all}"]
from_port = 8080
protocol = "tcp"
to_port = 8080
[[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]]
cidr_blocks = ["${var.ip_all}"]
from_port = 80
protocol = "tcp"
to_port = 80
[[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]]
cidr_blocks = [
"${var.ip_kevin}",
"${var.ip_marius}",
"${var.ip_tim}",
]
from_port = 22
protocol = "tcp"
to_port = 22
[[install.playbooks]]
hosts = "infra-tusd-server"
name = "Install infra-tusd-server"
roles = [
"{{{init.paths.roles_dir}}}/apt/v1.0.0",
]
[install.playbooks.vars]
apt_packages = [
"apg",
"build-essential",
"curl",
"git-core",
"htop",
"iotop",
"libpcre3",
"logtail",
"mlocate",
"mtr",
"psmisc",
"telnet",
"vim",
"wget",
]
[[install.playbooks.tasks]]
action = "lineinfile dest=/home/ubuntu/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\""
name = "Common | Add convenience shortcut wtf"
[[setup.playbooks]]
hosts = "infra-tusd-server"
name = "Setup infra-tusd"
[[setup.playbooks.tasks]]
hostname = "name={{lookup('env', 'FREY_DOMAIN')}}"
name = "infra-tusd | Set hostname"
[[setup.playbooks.tasks]]
file = "path=/srv/tusd/shared/logs state=directory owner=www-data group=www-data mode=0755 recurse=yes"
name = "infra-tusd | Create shared log dir"
[[setup.playbooks.tasks]]
file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes"
name = "infra-tusd | Create tusd data dir"
[[setup.playbooks.tasks]]
action = "template src=templates/upstart-tusd.conf.j2 dest=/etc/init/tusd.conf"
name = "infra-tusd | Install upstart file"
[[deploy.playbooks]]
hosts = "infra-tusd-server"
name = "Deploy infra-tusd"
roles = [
"{{{init.paths.roles_dir}}}/deploy/v1.4.0",
]
[deploy.playbooks.vars]
ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz"
ansistrano_deploy_to = "/srv/tusd"
ansistrano_deploy_via = "download_unarchive"
ansistrano_npm = no
ansistrano_owner = "www-data"
ansistrano_group = "www-data"
[[deploy.playbooks.tasks]]
copy = "src=../env.sh dest=/srv/tusd/current/env.sh mode=0600 owner=root group=root"
name = "infra-tusd | Upload environment"
[[deploy.playbooks.tasks]]
name = "tusd | Set file attributes"
file = "path=/srv/tusd/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data"
[[restart.playbooks]]
hosts = "infra-tusd-server"
name = "Restart infra-tusd"
[[restart.playbooks.tasks]]
shell = "iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080"
name = "infra-tusd | Redirect HTTP traffic to tusd"
[[restart.playbooks.tasks]]
action = "service name=tusd state=restarted"
name = "infra-tusd | Restart"

3
.infra/README.md Normal file
View File

@ -0,0 +1,3 @@
This folder is charge of launching master.tus.io via [Frey](https://github.com/kvz/frey).
You could re-use bits of this, but you could also run tusd any other way you want.
It's not a requirement for running a tusd server.

10
.infra/env.example.sh Normal file
View File

@ -0,0 +1,10 @@
# Rename this file to env.sh, it will be kept out of Git.
# So suitable for adding secret keys and such
# export DEBUG="frey:*"
# export FREY_DOMAIN="master.tus.io"
# export FREY_ENCRYPTION_SECRET="***"
# source env.sh
# travis encrypt --add env.global "FREY_DOMAIN=${FREY_DOMAIN}"
# travis encrypt --add env.global "FREY_ENCRYPTION_SECRET=${FREY_ENCRYPTION_SECRET}"

View File

@ -0,0 +1,5 @@
source env.sh
# export FREY_AWS_ACCESS_KEY="***"
# export FREY_AWS_SECRET_KEY="***"
# export FREY_AWS_ZONE_ID="***"

Binary file not shown.

View File

View File

@ -0,0 +1,25 @@
stop on runlevel [016]
start on (started networking)
# The respawn limits function as follows: If the process is respawned
# more than count times within an interval of timeout seconds,
# the process will be stopped automatically, and not restarted.
# Unless set explicitly, the limit defaults to 10 times within 5 seconds.
# http://upstart.ubuntu.com/wiki/Stanzas#respawn_limit
respawn
respawn limit 10 5
limit nofile 32768 32768
pre-stop exec status tusd | grep -q "stop/waiting" && initctl emit --no-wait stopped JOB=tusd || true
script
set -e
set -x
mkfifo /tmp/tusd-log-fifo
( logger -t tusd </tmp/tusd-log-fifo & )
exec >/tmp/tusd-log-fifo
rm /tmp/tusd-log-fifo
exec bash -c "cd /srv/tusd/current/tusd_linux_amd64/ \
&& exec sudo -EHu www-data ./tusd -port=8080 -dir=/mnt/tusd-data -store-size=10737418240"
end script

View File

@ -17,7 +17,7 @@ matrix:
- go: tip - go: tip
install: install:
- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \))
- rsync -r ./vendor/ $GOPATH/src - rsync -r ./vendor/ $GOPATH/src
script: script:
@ -50,3 +50,6 @@ deploy:
tags: true tags: true
go: 1.5 go: 1.5
repo: tus/tusd repo: tus/tusd
after_deploy:
- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --projectDir .infra) else echo "Skipping deploy for non-master/PRs"; fi

19
Makefile Normal file
View File

@ -0,0 +1,19 @@
PHONY: frey
frey:
@npm install --global frey@0.3.12
PHONY: deploy
deploy:
@cd .infra && source env.sh && frey setup
PHONY: launch
launch:
@cd .infra && source env.infra.sh && frey infra
PHONY: console
console:
@cd .infra && source env.sh && frey remote
PHONY: deploy-localfrey
deploy-localfrey:
@cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js setup