From e54850a3aa127435cf6b9a3655115bcda2eed42c Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 13:37:19 +0200 Subject: [PATCH 01/25] Add Frey support Squashed commit of the following: commit 9071377570a97d599f7fc769f4d26b45d03020cb Author: Kevin van Zonneveld Date: Tue Mar 29 13:36:57 2016 +0200 Remove useless env code commit 529aed3535d0159dd7291532ebb6cfcc9ae401c6 Author: Kevin van Zonneveld Date: Tue Mar 29 13:31:38 2016 +0200 Update state, add launch target commit fde533f9f6cc896a7e2abd7e68320d806d90ba27 Author: Kevin van Zonneveld Date: Tue Mar 29 11:07:35 2016 +0200 First swing at moving Frey to tusd --- .gitignore | 5 + .infra/Frey-state-terraform.tfstate | 119 +++++++++++++++ .infra/Frey-state-terraform.tfstate.backup | 119 +++++++++++++++ .infra/Freyfile.toml | 159 +++++++++++++++++++++ .infra/README.md | 3 + .infra/env.example.sh | 10 ++ .infra/env.infra.example.sh | 5 + .infra/ssh/frey-infra-tusd.pem.cast5 | Bin 0 -> 1696 bytes .infra/ssh/frey-infra-tusd.pub | 0 .infra/templates/upstart-tusd.conf.j2 | 25 ++++ .travis.yml | 5 +- Makefile | 19 +++ 12 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 .infra/Frey-state-terraform.tfstate create mode 100644 .infra/Frey-state-terraform.tfstate.backup create mode 100644 .infra/Freyfile.toml create mode 100644 .infra/README.md create mode 100644 .infra/env.example.sh create mode 100644 .infra/env.infra.example.sh create mode 100644 .infra/ssh/frey-infra-tusd.pem.cast5 create mode 100644 .infra/ssh/frey-infra-tusd.pub create mode 100644 .infra/templates/upstart-tusd.conf.j2 create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 63d6663..13506b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ tusd/data cover.out data/ +.infra/env.sh +.infra/env.infra.sh +.infra/ssh/infra-tusd.pem +.infra/Frey-residu* +.infra/ssh/frey-infra-tusd.pem diff --git a/.infra/Frey-state-terraform.tfstate b/.infra/Frey-state-terraform.tfstate new file mode 100644 index 0000000..a711876 --- /dev/null +++ b/.infra/Frey-state-terraform.tfstate @@ -0,0 +1,119 @@ +{ + "version": 1, + "serial": 20, + "modules": [ + { + "path": [ + "root" + ], + "outputs": { + "endpoint": "http://master.tus.io:80/", + "public_address": "ec2-54-161-247-36.compute-1.amazonaws.com", + "public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com" + }, + "resources": { + "aws_instance.infra-tusd-server": { + "type": "aws_instance", + "primary": { + "id": "i-d59cff7e", + "attributes": { + "ami": "ami-9bce7af0", + "availability_zone": "us-east-1e", + "ebs_block_device.#": "0", + "ebs_optimized": "false", + "ephemeral_block_device.#": "0", + "iam_instance_profile": "", + "id": "i-d59cff7e", + "instance_state": "running", + "instance_type": "c3.large", + "key_name": "infra-tusd", + "monitoring": "false", + "private_dns": "ip-10-167-178-105.ec2.internal", + "private_ip": "10.167.178.105", + "public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com", + "public_ip": "54.161.247.36", + "root_block_device.#": "1", + "root_block_device.0.delete_on_termination": "true", + "root_block_device.0.iops": "24", + "root_block_device.0.volume_size": "8", + "root_block_device.0.volume_type": "gp2", + "security_groups.#": "1", + "security_groups.1246499019": "fw-infra-tusd-main", + "source_dest_check": "true", + "subnet_id": "", + "tags.#": "1", + "tags.Name": "master.tus.io", + "tenancy": "default", + "vpc_security_group_ids.#": "0" + }, + "meta": { + "schema_version": "1" + } + } + }, + "aws_route53_record.www": { + "type": "aws_route53_record", + "depends_on": [ + "aws_instance.infra-tusd-server" + ], + "primary": { + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "attributes": { + "failover": "", + "fqdn": "master.tus.io", + "health_check_id": "", + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "name": "master.tus.io", + "records.#": "1", + "records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com", + "set_identifier": "", + "ttl": "300", + "type": "CNAME", + "weight": "-1", + "zone_id": "Z3IT8X6U91XY1P" + } + } + }, + "aws_security_group.fw-infra-tusd-main": { + "type": "aws_security_group", + "primary": { + "id": "sg-2ff78c42", + "attributes": { + "description": "Infra tusd", + "egress.#": "0", + "id": "sg-2ff78c42", + "ingress.#": "3", + "ingress.2214680975.cidr_blocks.#": "1", + "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", + "ingress.2214680975.from_port": "80", + "ingress.2214680975.protocol": "tcp", + "ingress.2214680975.security_groups.#": "0", + "ingress.2214680975.self": "false", + "ingress.2214680975.to_port": "80", + "ingress.516175195.cidr_blocks.#": "1", + "ingress.516175195.cidr_blocks.0": "0.0.0.0/0", + "ingress.516175195.from_port": "8080", + "ingress.516175195.protocol": "tcp", + "ingress.516175195.security_groups.#": "0", + "ingress.516175195.self": "false", + "ingress.516175195.to_port": "8080", + "ingress.614077637.cidr_blocks.#": "3", + "ingress.614077637.cidr_blocks.0": "62.163.187.106/32", + "ingress.614077637.cidr_blocks.1": "84.146.0.0/16", + "ingress.614077637.cidr_blocks.2": "24.134.75.132/32", + "ingress.614077637.from_port": "22", + "ingress.614077637.protocol": "tcp", + "ingress.614077637.security_groups.#": "0", + "ingress.614077637.self": "false", + "ingress.614077637.to_port": "22", + "name": "fw-infra-tusd-main", + "owner_id": "402421253186", + "tags.#": "0", + "vpc_id": "" + } + } + } + } + } + ] +} diff --git a/.infra/Frey-state-terraform.tfstate.backup b/.infra/Frey-state-terraform.tfstate.backup new file mode 100644 index 0000000..a711876 --- /dev/null +++ b/.infra/Frey-state-terraform.tfstate.backup @@ -0,0 +1,119 @@ +{ + "version": 1, + "serial": 20, + "modules": [ + { + "path": [ + "root" + ], + "outputs": { + "endpoint": "http://master.tus.io:80/", + "public_address": "ec2-54-161-247-36.compute-1.amazonaws.com", + "public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com" + }, + "resources": { + "aws_instance.infra-tusd-server": { + "type": "aws_instance", + "primary": { + "id": "i-d59cff7e", + "attributes": { + "ami": "ami-9bce7af0", + "availability_zone": "us-east-1e", + "ebs_block_device.#": "0", + "ebs_optimized": "false", + "ephemeral_block_device.#": "0", + "iam_instance_profile": "", + "id": "i-d59cff7e", + "instance_state": "running", + "instance_type": "c3.large", + "key_name": "infra-tusd", + "monitoring": "false", + "private_dns": "ip-10-167-178-105.ec2.internal", + "private_ip": "10.167.178.105", + "public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com", + "public_ip": "54.161.247.36", + "root_block_device.#": "1", + "root_block_device.0.delete_on_termination": "true", + "root_block_device.0.iops": "24", + "root_block_device.0.volume_size": "8", + "root_block_device.0.volume_type": "gp2", + "security_groups.#": "1", + "security_groups.1246499019": "fw-infra-tusd-main", + "source_dest_check": "true", + "subnet_id": "", + "tags.#": "1", + "tags.Name": "master.tus.io", + "tenancy": "default", + "vpc_security_group_ids.#": "0" + }, + "meta": { + "schema_version": "1" + } + } + }, + "aws_route53_record.www": { + "type": "aws_route53_record", + "depends_on": [ + "aws_instance.infra-tusd-server" + ], + "primary": { + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "attributes": { + "failover": "", + "fqdn": "master.tus.io", + "health_check_id": "", + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "name": "master.tus.io", + "records.#": "1", + "records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com", + "set_identifier": "", + "ttl": "300", + "type": "CNAME", + "weight": "-1", + "zone_id": "Z3IT8X6U91XY1P" + } + } + }, + "aws_security_group.fw-infra-tusd-main": { + "type": "aws_security_group", + "primary": { + "id": "sg-2ff78c42", + "attributes": { + "description": "Infra tusd", + "egress.#": "0", + "id": "sg-2ff78c42", + "ingress.#": "3", + "ingress.2214680975.cidr_blocks.#": "1", + "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", + "ingress.2214680975.from_port": "80", + "ingress.2214680975.protocol": "tcp", + "ingress.2214680975.security_groups.#": "0", + "ingress.2214680975.self": "false", + "ingress.2214680975.to_port": "80", + "ingress.516175195.cidr_blocks.#": "1", + "ingress.516175195.cidr_blocks.0": "0.0.0.0/0", + "ingress.516175195.from_port": "8080", + "ingress.516175195.protocol": "tcp", + "ingress.516175195.security_groups.#": "0", + "ingress.516175195.self": "false", + "ingress.516175195.to_port": "8080", + "ingress.614077637.cidr_blocks.#": "3", + "ingress.614077637.cidr_blocks.0": "62.163.187.106/32", + "ingress.614077637.cidr_blocks.1": "84.146.0.0/16", + "ingress.614077637.cidr_blocks.2": "24.134.75.132/32", + "ingress.614077637.from_port": "22", + "ingress.614077637.protocol": "tcp", + "ingress.614077637.security_groups.#": "0", + "ingress.614077637.self": "false", + "ingress.614077637.to_port": "22", + "name": "fw-infra-tusd-main", + "owner_id": "402421253186", + "tags.#": "0", + "vpc_id": "" + } + } + } + } + } + ] +} diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml new file mode 100644 index 0000000..ef7f385 --- /dev/null +++ b/.infra/Freyfile.toml @@ -0,0 +1,159 @@ +[global] + appname = "infra-tusd" + +[global.ssh] + key_dir = "./ssh" + +[global.ansiblecfg.privilege_escalation] + become = true + +[global.ansiblecfg.defaults] + host_key_checking = "False" + +[infra.provider.aws] + access_key = "${var.FREY_AWS_ACCESS_KEY}" + region = "us-east-1" + secret_key = "${var.FREY_AWS_SECRET_KEY}" + +[infra.variable.ami.default] + us-east-1 = "ami-9bce7af0" +[infra.variable.ip_all] + default = "0.0.0.0/0" +[infra.variable.ip_kevin] + default = "62.163.187.106/32" +[infra.variable.ip_marius] + default = "84.146.0.0/16" +[infra.variable.ip_tim] + default = "24.134.75.132/32" +[infra.variable.region] + default = "us-east-1" + +[infra.output.public_address] + value = "${aws_instance.infra-tusd-server.0.public_dns}" +[infra.output.public_addresses] + value = "${join(\"\n\", aws_instance.infra-tusd-server.*.public_dns)}" +[infra.output.endpoint] + value = "http://${aws_route53_record.www.name}:80/" + +[infra.resource.aws_instance.infra-tusd-server] + ami = "${lookup(var.ami, var.region)}" + instance_type = "c3.large" + key_name = "infra-tusd" + security_groups = ["fw-infra-tusd-main"] + [infra.resource.aws_instance.infra-tusd-server.connection] + key_file = "{{{config.global.ssh.privatekey_file}}}" + user = "{{{config.global.ssh.user}}}" + [infra.resource.aws_instance.infra-tusd-server.tags] + Name = "${var.FREY_DOMAIN}" + +[infra.resource.aws_route53_record.www] + name = "${var.FREY_DOMAIN}" + records = ["${aws_instance.infra-tusd-server.public_dns}"] + ttl = "300" + type = "CNAME" + zone_id = "${var.FREY_AWS_ZONE_ID}" + +[infra.resource.aws_security_group.fw-infra-tusd-main] + description = "Infra tusd" + name = "fw-infra-tusd-main" + [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + cidr_blocks = ["${var.ip_all}"] + from_port = 8080 + protocol = "tcp" + to_port = 8080 + [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + cidr_blocks = ["${var.ip_all}"] + from_port = 80 + protocol = "tcp" + to_port = 80 + [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + cidr_blocks = [ + "${var.ip_kevin}", + "${var.ip_marius}", + "${var.ip_tim}", + ] + from_port = 22 + protocol = "tcp" + to_port = 22 + +[[install.playbooks]] + hosts = "infra-tusd-server" + name = "Install infra-tusd-server" + roles = [ + "{{{init.paths.roles_dir}}}/apt/v1.0.0", + ] + [install.playbooks.vars] + apt_packages = [ + "apg", + "build-essential", + "curl", + "git-core", + "htop", + "iotop", + "libpcre3", + "logtail", + "mlocate", + "mtr", + "psmisc", + "telnet", + "vim", + "wget", + ] + + [[install.playbooks.tasks]] + action = "lineinfile dest=/home/ubuntu/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\"" + name = "Common | Add convenience shortcut wtf" + +[[setup.playbooks]] + hosts = "infra-tusd-server" + name = "Setup infra-tusd" + + [[setup.playbooks.tasks]] + hostname = "name={{lookup('env', 'FREY_DOMAIN')}}" + name = "infra-tusd | Set hostname" + + [[setup.playbooks.tasks]] + file = "path=/srv/tusd/shared/logs state=directory owner=www-data group=www-data mode=0755 recurse=yes" + name = "infra-tusd | Create shared log dir" + + [[setup.playbooks.tasks]] + file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes" + name = "infra-tusd | Create tusd data dir" + + [[setup.playbooks.tasks]] + action = "template src=templates/upstart-tusd.conf.j2 dest=/etc/init/tusd.conf" + name = "infra-tusd | Install upstart file" + +[[deploy.playbooks]] + hosts = "infra-tusd-server" + name = "Deploy infra-tusd" + roles = [ + "{{{init.paths.roles_dir}}}/deploy/v1.4.0", + ] + [deploy.playbooks.vars] + ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz" + ansistrano_deploy_to = "/srv/tusd" + ansistrano_deploy_via = "download_unarchive" + ansistrano_npm = no + ansistrano_owner = "www-data" + ansistrano_group = "www-data" + + [[deploy.playbooks.tasks]] + copy = "src=../env.sh dest=/srv/tusd/current/env.sh mode=0600 owner=root group=root" + name = "infra-tusd | Upload environment" + + [[deploy.playbooks.tasks]] + name = "tusd | Set file attributes" + file = "path=/srv/tusd/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" + +[[restart.playbooks]] + hosts = "infra-tusd-server" + name = "Restart infra-tusd" + + [[restart.playbooks.tasks]] + shell = "iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080" + name = "infra-tusd | Redirect HTTP traffic to tusd" + + [[restart.playbooks.tasks]] + action = "service name=tusd state=restarted" + name = "infra-tusd | Restart" diff --git a/.infra/README.md b/.infra/README.md new file mode 100644 index 0000000..eac03a4 --- /dev/null +++ b/.infra/README.md @@ -0,0 +1,3 @@ +This folder is charge of launching master.tus.io via [Frey](https://github.com/kvz/frey). +You could re-use bits of this, but you could also run tusd any other way you want. +It's not a requirement for running a tusd server. diff --git a/.infra/env.example.sh b/.infra/env.example.sh new file mode 100644 index 0000000..0fe3534 --- /dev/null +++ b/.infra/env.example.sh @@ -0,0 +1,10 @@ +# Rename this file to env.sh, it will be kept out of Git. +# So suitable for adding secret keys and such + +# export DEBUG="frey:*" +# export FREY_DOMAIN="master.tus.io" +# export FREY_ENCRYPTION_SECRET="***" + +# source env.sh +# travis encrypt --add env.global "FREY_DOMAIN=${FREY_DOMAIN}" +# travis encrypt --add env.global "FREY_ENCRYPTION_SECRET=${FREY_ENCRYPTION_SECRET}" diff --git a/.infra/env.infra.example.sh b/.infra/env.infra.example.sh new file mode 100644 index 0000000..e961ab3 --- /dev/null +++ b/.infra/env.infra.example.sh @@ -0,0 +1,5 @@ +source env.sh + +# export FREY_AWS_ACCESS_KEY="***" +# export FREY_AWS_SECRET_KEY="***" +# export FREY_AWS_ZONE_ID="***" diff --git a/.infra/ssh/frey-infra-tusd.pem.cast5 b/.infra/ssh/frey-infra-tusd.pem.cast5 new file mode 100644 index 0000000000000000000000000000000000000000..215171ffe7f8352fe577c2ad0ff7d012dd7a14ca GIT binary patch literal 1696 zcmV;R24DHNEgP#BlHP0|6U5}`zeb9;6Jf4Q4dYZJ@^X*$Y5%dWBec(tl3~fXPf0?S zFsJ16MS=VccQQ_dWYmA^x^iNB#3@=|TwCP_fha{QbM$x*Ys-)t zYWHL!4zh&M0?P*BiBQPPsh(%QU@3)tO7Z@q#{UmwL7wZ{`>hp*<}x|zeDBTv(kf)5 z+dR(P2%04mRDH(;uKnOv^W%H6(g2yE4O(ZOCA?ET!u>y5&e1?4c}V^;I?WIJhg2N@ zS$D9fA#w_)U~xg{QG7J}9<(v>)7W9Fc;)g3x;Ld>FVG>;U=)HBGdB*W8s??MtD^TR zvW$lkeBGZVgtI5EjoRzteIIB2V=L$*(#l?|_P6qlFxzxur1iK?JQtUnv_vti{3BJE zU&J#iEOU!E$eicY-owR@FSETKR6_$XQpvVO4@X59W z@S(>A-_kX`Pi#ioL*S6S!4V}9Z^*8~!LjY4;#fFPl&h7#lq6Pxm*QXY|57KareE9V z`3=c%3AQUs-n}@Iksm>eYPTmuaw?JttWz)gdBofiE3HvLHsl%=dA4(*sN%Y;7d!Fr zCTFY(9V$i-?~7sZPjhdcZxJptv6&hmHTpD~+ijeY#d}r1l^tq|$|L1jf`W+}QuNU{ zOoMibAs?mLyGLd^S14WRjV)SRBn~9|#B}Pk0&^KnGfRv|xy)KM2Wk8-O`nHQoWUuh zsofbgUfkBR-q?HUc^Z0}XN^L?r?fLLBX{e62R%SO(40+fjo)`$ObZq_8rL`-oV@O# z>&${SyIyu$NjKddCuK7d8HzsC6Ml@iQl=A!NnSmFt+BRH5?Hr*H!2(eIhL!^2=1$GFUeEueW_0j2B{J#!i^s<_Ss zew?coc~_=cGi<9L6$q34l>(b#J&O3uu~!@v9MHl1)g#NqSGJOdS_chJw6&@P%rDPS z!6GJ?1A(UlRzW($po(*>HP!K}QGa`f{gg?Gzt zyM=c1&&By{GBXIn^!!40vKF+ag*2tf zy=U0G?{)%l_2c?HmN6Gu;ABgkzAc)*85zE`l8a3J;m0B|()a zsK)YWJW$aNU~`Ba>#=T<5?o4Kx9;8q-K@SR0XKVWOp(s$yJiI7}90q#+zBpTD7{PiivJa6Ik$rJ6C(88v?rY|y1$NDr%# z-=diVG?bz@-dfWa4xwA|-(Nla3R_fvmzGfiMwNEw#WpG_W!C^|h`$lS6=D8FwNI9H zu3Okp!>Bq)&uJjZ^OALCLA|WY7~h#%VKl*{A9ud0PQ#DQz@9)`pFL~%93>xb{NHJ7 zH`R;}pqM`R{sCP3leu!b1{H>hnE`865|3BZ1$K;X!qh}29-j7)Hx`vcoK9!3)G2Zv q;Md8^J1Zc(u%b7kGm?rZMRK-qi1dvvc&}$drcD(n4)V|b<5&CIwOKg; literal 0 HcmV?d00001 diff --git a/.infra/ssh/frey-infra-tusd.pub b/.infra/ssh/frey-infra-tusd.pub new file mode 100644 index 0000000..e69de29 diff --git a/.infra/templates/upstart-tusd.conf.j2 b/.infra/templates/upstart-tusd.conf.j2 new file mode 100644 index 0000000..5a74f75 --- /dev/null +++ b/.infra/templates/upstart-tusd.conf.j2 @@ -0,0 +1,25 @@ +stop on runlevel [016] +start on (started networking) + +# The respawn limits function as follows: If the process is respawned +# more than count times within an interval of timeout seconds, +# the process will be stopped automatically, and not restarted. +# Unless set explicitly, the limit defaults to 10 times within 5 seconds. +# http://upstart.ubuntu.com/wiki/Stanzas#respawn_limit +respawn +respawn limit 10 5 + +limit nofile 32768 32768 + +pre-stop exec status tusd | grep -q "stop/waiting" && initctl emit --no-wait stopped JOB=tusd || true + +script + set -e + set -x + mkfifo /tmp/tusd-log-fifo + ( logger -t tusd /tmp/tusd-log-fifo + rm /tmp/tusd-log-fifo + exec bash -c "cd /srv/tusd/current/tusd_linux_amd64/ \ + && exec sudo -EHu www-data ./tusd -port=8080 -dir=/mnt/tusd-data -store-size=10737418240" +end script diff --git a/.travis.yml b/.travis.yml index 13f0727..d10ea33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: - go: tip install: -- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) +- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - rsync -r ./vendor/ $GOPATH/src script: @@ -50,3 +50,6 @@ deploy: tags: true go: 1.5 repo: tus/tusd + +after_deploy: + - if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --projectDir .infra) else echo "Skipping deploy for non-master/PRs"; fi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7e5c818 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +PHONY: frey +frey: + @npm install --global frey@0.3.12 + +PHONY: deploy +deploy: + @cd .infra && source env.sh && frey setup + +PHONY: launch +launch: + @cd .infra && source env.infra.sh && frey infra + +PHONY: console +console: + @cd .infra && source env.sh && frey remote + +PHONY: deploy-localfrey +deploy-localfrey: + @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js setup From 43653b6d014605f6a8ef78fbaa967e9a5bd0a467 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 13:38:20 +0200 Subject: [PATCH 02/25] Travis whitespace --- .travis.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index d10ea33..5e7814c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,30 +5,25 @@ go: - 1.5 - 1.6rc2 - tip - sudo: false - env: global: - GO15VENDOREXPERIMENT=1 - matrix: allow_failures: - go: tip - install: -- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) +- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" + -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - rsync -r ./vendor/ $GOPATH/src - script: - go test $PACKAGES - before_deploy: - export GOROOT_BOOTSTRAP=$GOROOT - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" -- goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" - +- goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG + -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" deploy: provider: releases api_key: @@ -45,11 +40,11 @@ deploy: - snapshot/tusd_windows_386.zip - snapshot/tusd_windows_amd64.zip skip_cleanup: true - on: tags: true go: 1.5 repo: tus/tusd - after_deploy: - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --projectDir .infra) else echo "Skipping deploy for non-master/PRs"; fi +- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; + then (make frey && frey setup --projectDir .infra) else echo "Skipping deploy for + non-master/PRs"; fi From c549faafbce07a4b35d4043078e2cf4986281d65 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 13:38:28 +0200 Subject: [PATCH 03/25] Add Frey secrets --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5e7814c..4e2068b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ sudo: false env: global: - GO15VENDOREXPERIMENT=1 + - secure: Ls76vWCzitRPJ4gyIAj7SdAUkbp2hhYd0zsDjHXriZA+KmlrZLsr4L3QcOaX+MMjMyEURZMS43VVaygP6Y8oF5yv4wYBHKgYh0ZW96pWrIS++0B5lo+2U+JZtTKAfLnOLHUQfLL5h4fg9+oa4mlcuH+3FvaCwO0TJjLU1b0q3Fc= + - secure: aAfgsISNfOC/8FDZSctCJr/JnxxnPkd1o0Zcqc+mTw6NXUa4T3o+aqbaY/VK5lk7Pru9exs5j+HdaeGhQrLMZJca803Sg1r0m68+h/2bS58p9zjyPk7aULCSAF7EUlqT041QQ5UBJV7gpIxFW1nyD6vL0ZBW1wA1k1PpxTjznPA= matrix: allow_failures: - go: tip From f3437a7ec5c4ca6348743bbd3a611caf86008fad Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 13:42:55 +0200 Subject: [PATCH 04/25] Temporarily also deploy non-tagged commits to test frey deploy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4e2068b..528e61a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ deploy: - snapshot/tusd_windows_amd64.zip skip_cleanup: true on: - tags: true + # tags: true go: 1.5 repo: tus/tusd after_deploy: From 4a30a6ad37447bf87a24687d21f1e8b7a1154083 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 13:54:21 +0200 Subject: [PATCH 05/25] Frey 0.3.12 is not released yet --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7e5c818..464b940 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PHONY: frey frey: - @npm install --global frey@0.3.12 + @npm install --global frey@0.3.11 PHONY: deploy deploy: From 1f7d84f202dc8e6cbac2438b78bd360d35411e8e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:02:33 +0200 Subject: [PATCH 06/25] Upgrade to Frey 0.3.12 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 464b940..7e5c818 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PHONY: frey frey: - @npm install --global frey@0.3.11 + @npm install --global frey@0.3.12 PHONY: deploy deploy: From 2d30a16ffca5bb4c492f6e736acaa839fc337a45 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:17:15 +0200 Subject: [PATCH 07/25] Move code-deploy to before release-deploy --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 528e61a..0d79f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,7 @@ matrix: allow_failures: - go: tip install: -- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" - -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) +- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - rsync -r ./vendor/ $GOPATH/src script: - go test $PACKAGES @@ -24,8 +23,8 @@ before_deploy: - export GOROOT_BOOTSTRAP=$GOROOT - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" -- goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG - -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" +- goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" +- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --projectDir .infra); else echo "Skipping deploy fornon-master/PRs"; fi deploy: provider: releases api_key: @@ -46,7 +45,3 @@ deploy: # tags: true go: 1.5 repo: tus/tusd -after_deploy: -- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; - then (make frey && frey setup --projectDir .infra) else echo "Skipping deploy for - non-master/PRs"; fi From 373001127cafe885f9d86681000d3525486daba0 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:31:33 +0200 Subject: [PATCH 08/25] Speed up Travis build by caching apt, Frey tools and gimme --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0d79f1d..7d7c563 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,11 @@ go: - 1.6rc2 - tip sudo: false +cache: + apt: true + directories: + - $HOME/.gimme + - $HOME/.frey env: global: - GO15VENDOREXPERIMENT=1 From 68ed4ebcf0f929ac15c46cd4db1a2a579c4c1c49 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:49:46 +0200 Subject: [PATCH 09/25] Refactor Freyfile --- .infra/Freyfile.toml | 90 +++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index ef7f385..3c95cc7 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -1,5 +1,6 @@ [global] appname = "infra-tusd" + approot = "/srv/tusd" [global.ssh] key_dir = "./ssh" @@ -8,7 +9,7 @@ become = true [global.ansiblecfg.defaults] - host_key_checking = "False" + host_key_checking = false [infra.provider.aws] access_key = "${var.FREY_AWS_ACCESS_KEY}" @@ -36,36 +37,36 @@ value = "http://${aws_route53_record.www.name}:80/" [infra.resource.aws_instance.infra-tusd-server] - ami = "${lookup(var.ami, var.region)}" - instance_type = "c3.large" - key_name = "infra-tusd" + ami = "${lookup(var.ami, var.region)}" + instance_type = "c3.large" + key_name = "infra-tusd" security_groups = ["fw-infra-tusd-main"] [infra.resource.aws_instance.infra-tusd-server.connection] key_file = "{{{config.global.ssh.privatekey_file}}}" - user = "{{{config.global.ssh.user}}}" + user = "{{{config.global.ssh.user}}}" [infra.resource.aws_instance.infra-tusd-server.tags] Name = "${var.FREY_DOMAIN}" [infra.resource.aws_route53_record.www] - name = "${var.FREY_DOMAIN}" + name = "${var.FREY_DOMAIN}" records = ["${aws_instance.infra-tusd-server.public_dns}"] - ttl = "300" - type = "CNAME" + ttl = "300" + type = "CNAME" zone_id = "${var.FREY_AWS_ZONE_ID}" [infra.resource.aws_security_group.fw-infra-tusd-main] description = "Infra tusd" - name = "fw-infra-tusd-main" + name = "fw-infra-tusd-main" [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] cidr_blocks = ["${var.ip_all}"] - from_port = 8080 - protocol = "tcp" - to_port = 8080 + from_port = 8080 + protocol = "tcp" + to_port = 8080 [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] cidr_blocks = ["${var.ip_all}"] - from_port = 80 - protocol = "tcp" - to_port = 80 + from_port = 80 + protocol = "tcp" + to_port = 80 [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] cidr_blocks = [ "${var.ip_kevin}", @@ -73,16 +74,15 @@ "${var.ip_tim}", ] from_port = 22 - protocol = "tcp" - to_port = 22 + protocol = "tcp" + to_port = 22 [[install.playbooks]] hosts = "infra-tusd-server" - name = "Install infra-tusd-server" - roles = [ - "{{{init.paths.roles_dir}}}/apt/v1.0.0", - ] - [install.playbooks.vars] + name = "Install infra-tusd-server" + + [[install.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/apt/v1.0.0" apt_packages = [ "apg", "build-essential", @@ -101,19 +101,37 @@ ] [[install.playbooks.tasks]] - action = "lineinfile dest=/home/ubuntu/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\"" - name = "Common | Add convenience shortcut wtf" + lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" + name = "Common | Add convenience shortcut wtf" + + [[install.playbooks.tasks]] + lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"cd {{{config.global.approot}}}/current || true\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" + name = "Common | Install login" + + [[install.playbooks.tasks]] + name = "Common | Set motd" + copy = "content='Welcome to {{lookup('env', 'FREY_DOMAIN')}}' dest=/etc/motd owner=root group=root mode=0644 backup=yes" + + [[install.playbooks.tasks]] + name = "Common | Set timezone variables" + copy = "content='Etc/UTC' dest=/etc/timezone owner=root group=root mode=0644 backup=yes" + notify = [ "Common | Update timezone" ] + + [[install.playbooks.handlers]] + name = "Common | Update timezone" + command = "dpkg-reconfigure --frontend noninteractive tzdata" + [[setup.playbooks]] hosts = "infra-tusd-server" name = "Setup infra-tusd" - [[setup.playbooks.tasks]] - hostname = "name={{lookup('env', 'FREY_DOMAIN')}}" - name = "infra-tusd | Set hostname" + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/fqdn/v1.0.0" + fqdn = "{{lookup('env', 'FREY_DOMAIN')}}" [[setup.playbooks.tasks]] - file = "path=/srv/tusd/shared/logs state=directory owner=www-data group=www-data mode=0755 recurse=yes" + file = "path={{{config.global.approot}}}/shared/logs state=directory owner=www-data group=www-data mode=0755 recurse=yes" name = "infra-tusd | Create shared log dir" [[setup.playbooks.tasks]] @@ -127,24 +145,20 @@ [[deploy.playbooks]] hosts = "infra-tusd-server" name = "Deploy infra-tusd" - roles = [ - "{{{init.paths.roles_dir}}}/deploy/v1.4.0", - ] - [deploy.playbooks.vars] + + [[deploy.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/deploy/v1.4.0" ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz" - ansistrano_deploy_to = "/srv/tusd" + ansistrano_deploy_to = "{{{config.global.approot}}}" ansistrano_deploy_via = "download_unarchive" - ansistrano_npm = no - ansistrano_owner = "www-data" - ansistrano_group = "www-data" [[deploy.playbooks.tasks]] - copy = "src=../env.sh dest=/srv/tusd/current/env.sh mode=0600 owner=root group=root" + copy = "src=../env.sh dest={{{config.global.approot}}}/current/env.sh mode=0600 owner=root group=root" name = "infra-tusd | Upload environment" [[deploy.playbooks.tasks]] name = "tusd | Set file attributes" - file = "path=/srv/tusd/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" + file = "path={{{config.global.approot}}}/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" [[restart.playbooks]] hosts = "infra-tusd-server" From 7db4d51c1a84d87660b1a2241512913050d52533 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:59:15 +0200 Subject: [PATCH 10/25] Allow ubuntu group access to release --- .infra/Freyfile.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index 3c95cc7..9ab740d 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -151,6 +151,7 @@ ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz" ansistrano_deploy_to = "{{{config.global.approot}}}" ansistrano_deploy_via = "download_unarchive" + ansistrano_group = "ubuntu" [[deploy.playbooks.tasks]] copy = "src=../env.sh dest={{{config.global.approot}}}/current/env.sh mode=0600 owner=root group=root" From b95fc0615c25f233261f0cd8b8071d499062a5ac Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 15:59:51 +0200 Subject: [PATCH 11/25] No need to create shared log dir, Frey does this for us now --- .infra/Freyfile.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index 9ab740d..1bb94b3 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -130,10 +130,6 @@ role = "{{{init.paths.roles_dir}}}/fqdn/v1.0.0" fqdn = "{{lookup('env', 'FREY_DOMAIN')}}" - [[setup.playbooks.tasks]] - file = "path={{{config.global.approot}}}/shared/logs state=directory owner=www-data group=www-data mode=0755 recurse=yes" - name = "infra-tusd | Create shared log dir" - [[setup.playbooks.tasks]] file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes" name = "infra-tusd | Create tusd data dir" From 98a6cb0cc549bb31d167b4e99f19b7d893c0e5b5 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 16:07:50 +0200 Subject: [PATCH 12/25] Always answer yes --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d7c563..72da320 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ before_deploy: - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" - goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" -- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --projectDir .infra); else echo "Skipping deploy fornon-master/PRs"; fi +- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --force-yes --projectDir .infra); else echo "Skipping deploy fornon-master/PRs"; fi deploy: provider: releases api_key: From 7d6a7beb8bdfc4ff937aa4616849cf7922099f13 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 16:20:11 +0200 Subject: [PATCH 13/25] Replace custom upstart template with Frey's upstart --- .infra/Freyfile.toml | 28 ++++++++++++++++++++++----- .infra/templates/upstart-tusd.conf.j2 | 25 ------------------------ 2 files changed, 23 insertions(+), 30 deletions(-) delete mode 100644 .infra/templates/upstart-tusd.conf.j2 diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index 1bb94b3..a18b318 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -13,7 +13,7 @@ [infra.provider.aws] access_key = "${var.FREY_AWS_ACCESS_KEY}" - region = "us-east-1" + region = "us-east-1" secret_key = "${var.FREY_AWS_SECRET_KEY}" [infra.variable.ami.default] @@ -126,6 +126,26 @@ hosts = "infra-tusd-server" name = "Setup infra-tusd" + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/upstart/v1.0.0" + upstart_command = "./tusd -port=8080 -dir=/mnt/tusd-data -store-size=10737418240" + upstart_description = "tusd server" + upstart_name = "{{{config.global.appname}}}" + upstart_pidfile_path = "{{{config.global.approot}}}/shared/{{{config.global.appname}}}.pid" + upstart_respawn = true + upstart_respawn_limit = true + upstart_runtime_root = "{{{config.global.approot}}}/current/tusd_linux_amd64/" + upstart_user = "www-data" + + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/rsyslog/v3.0.1" + [setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd] + # Send {{{config.global.appname}}} messages to a dedicated logfile + directives = [ "&~" ] + [[setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd.rules]] + rule = ":programname, startswith, \"{{{config.global.appname}}}\"" + logpath = "{{{config.global.approot}}}/shared/logs/{{{config.global.appname}}}.log" + [[setup.playbooks.roles]] role = "{{{init.paths.roles_dir}}}/fqdn/v1.0.0" fqdn = "{{lookup('env', 'FREY_DOMAIN')}}" @@ -134,15 +154,13 @@ file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes" name = "infra-tusd | Create tusd data dir" - [[setup.playbooks.tasks]] - action = "template src=templates/upstart-tusd.conf.j2 dest=/etc/init/tusd.conf" - name = "infra-tusd | Install upstart file" - [[deploy.playbooks]] hosts = "infra-tusd-server" name = "Deploy infra-tusd" [[deploy.playbooks.roles]] + # @todo On travis, we could directly use the generated + # /home/travis/gopath/src/github.com/tus/tusd/snapshot/tusd_linux_amd64.tar.gz role = "{{{init.paths.roles_dir}}}/deploy/v1.4.0" ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz" ansistrano_deploy_to = "{{{config.global.approot}}}" diff --git a/.infra/templates/upstart-tusd.conf.j2 b/.infra/templates/upstart-tusd.conf.j2 deleted file mode 100644 index 5a74f75..0000000 --- a/.infra/templates/upstart-tusd.conf.j2 +++ /dev/null @@ -1,25 +0,0 @@ -stop on runlevel [016] -start on (started networking) - -# The respawn limits function as follows: If the process is respawned -# more than count times within an interval of timeout seconds, -# the process will be stopped automatically, and not restarted. -# Unless set explicitly, the limit defaults to 10 times within 5 seconds. -# http://upstart.ubuntu.com/wiki/Stanzas#respawn_limit -respawn -respawn limit 10 5 - -limit nofile 32768 32768 - -pre-stop exec status tusd | grep -q "stop/waiting" && initctl emit --no-wait stopped JOB=tusd || true - -script - set -e - set -x - mkfifo /tmp/tusd-log-fifo - ( logger -t tusd /tmp/tusd-log-fifo - rm /tmp/tusd-log-fifo - exec bash -c "cd /srv/tusd/current/tusd_linux_amd64/ \ - && exec sudo -EHu www-data ./tusd -port=8080 -dir=/mnt/tusd-data -store-size=10737418240" -end script From 36bebff87c86771aa7718be161087aac2be09656 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 16:21:21 +0200 Subject: [PATCH 14/25] Rename infra-tusd -> tusd --- .gitignore | 4 +- .infra/Freyfile.toml | 52 +++++++++--------- ...fra-tusd.pem.cast5 => frey-tusd.pem.cast5} | Bin .../{frey-infra-tusd.pub => frey-tusd.pub} | 0 4 files changed, 28 insertions(+), 28 deletions(-) rename .infra/ssh/{frey-infra-tusd.pem.cast5 => frey-tusd.pem.cast5} (100%) rename .infra/ssh/{frey-infra-tusd.pub => frey-tusd.pub} (100%) diff --git a/.gitignore b/.gitignore index 13506b3..77f68eb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ cover.out data/ .infra/env.sh .infra/env.infra.sh -.infra/ssh/infra-tusd.pem +.infra/ssh/tusd.pem .infra/Frey-residu* -.infra/ssh/frey-infra-tusd.pem +.infra/ssh/frey-tusd.pem diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index a18b318..4b4031e 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -1,5 +1,5 @@ [global] - appname = "infra-tusd" + appname = "tusd" approot = "/srv/tusd" [global.ssh] @@ -30,44 +30,44 @@ default = "us-east-1" [infra.output.public_address] - value = "${aws_instance.infra-tusd-server.0.public_dns}" + value = "${aws_instance.tusd.0.public_dns}" [infra.output.public_addresses] - value = "${join(\"\n\", aws_instance.infra-tusd-server.*.public_dns)}" + value = "${join(\"\n\", aws_instance.tusd.*.public_dns)}" [infra.output.endpoint] value = "http://${aws_route53_record.www.name}:80/" -[infra.resource.aws_instance.infra-tusd-server] +[infra.resource.aws_instance.tusd] ami = "${lookup(var.ami, var.region)}" instance_type = "c3.large" - key_name = "infra-tusd" - security_groups = ["fw-infra-tusd-main"] - [infra.resource.aws_instance.infra-tusd-server.connection] + key_name = "tusd" + security_groups = ["fw-tusd-main"] + [infra.resource.aws_instance.tusd.connection] key_file = "{{{config.global.ssh.privatekey_file}}}" user = "{{{config.global.ssh.user}}}" - [infra.resource.aws_instance.infra-tusd-server.tags] + [infra.resource.aws_instance.tusd.tags] Name = "${var.FREY_DOMAIN}" [infra.resource.aws_route53_record.www] name = "${var.FREY_DOMAIN}" - records = ["${aws_instance.infra-tusd-server.public_dns}"] + records = ["${aws_instance.tusd.public_dns}"] ttl = "300" type = "CNAME" zone_id = "${var.FREY_AWS_ZONE_ID}" -[infra.resource.aws_security_group.fw-infra-tusd-main] +[infra.resource.aws_security_group.fw-tusd-main] description = "Infra tusd" - name = "fw-infra-tusd-main" - [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + name = "fw-tusd-main" + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] cidr_blocks = ["${var.ip_all}"] from_port = 8080 protocol = "tcp" to_port = 8080 - [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] cidr_blocks = ["${var.ip_all}"] from_port = 80 protocol = "tcp" to_port = 80 - [[infra.resource.aws_security_group.fw-infra-tusd-main.ingress]] + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] cidr_blocks = [ "${var.ip_kevin}", "${var.ip_marius}", @@ -78,8 +78,8 @@ to_port = 22 [[install.playbooks]] - hosts = "infra-tusd-server" - name = "Install infra-tusd-server" + hosts = "tusd" + name = "Install tusd" [[install.playbooks.roles]] role = "{{{init.paths.roles_dir}}}/apt/v1.0.0" @@ -123,8 +123,8 @@ [[setup.playbooks]] - hosts = "infra-tusd-server" - name = "Setup infra-tusd" + hosts = "tusd" + name = "Setup tusd" [[setup.playbooks.roles]] role = "{{{init.paths.roles_dir}}}/upstart/v1.0.0" @@ -152,11 +152,11 @@ [[setup.playbooks.tasks]] file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes" - name = "infra-tusd | Create tusd data dir" + name = "tusd | Create tusd data dir" [[deploy.playbooks]] - hosts = "infra-tusd-server" - name = "Deploy infra-tusd" + hosts = "tusd" + name = "Deploy tusd" [[deploy.playbooks.roles]] # @todo On travis, we could directly use the generated @@ -169,20 +169,20 @@ [[deploy.playbooks.tasks]] copy = "src=../env.sh dest={{{config.global.approot}}}/current/env.sh mode=0600 owner=root group=root" - name = "infra-tusd | Upload environment" + name = "tusd | Upload environment" [[deploy.playbooks.tasks]] name = "tusd | Set file attributes" file = "path={{{config.global.approot}}}/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" [[restart.playbooks]] - hosts = "infra-tusd-server" - name = "Restart infra-tusd" + hosts = "tusd" + name = "Restart tusd" [[restart.playbooks.tasks]] shell = "iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080" - name = "infra-tusd | Redirect HTTP traffic to tusd" + name = "tusd | Redirect HTTP traffic to tusd" [[restart.playbooks.tasks]] action = "service name=tusd state=restarted" - name = "infra-tusd | Restart" + name = "tusd | Restart" diff --git a/.infra/ssh/frey-infra-tusd.pem.cast5 b/.infra/ssh/frey-tusd.pem.cast5 similarity index 100% rename from .infra/ssh/frey-infra-tusd.pem.cast5 rename to .infra/ssh/frey-tusd.pem.cast5 diff --git a/.infra/ssh/frey-infra-tusd.pub b/.infra/ssh/frey-tusd.pub similarity index 100% rename from .infra/ssh/frey-infra-tusd.pub rename to .infra/ssh/frey-tusd.pub From d27fb04e5e345655646deacdf875da330f63cb5e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 16:43:12 +0200 Subject: [PATCH 15/25] Put env in subdir for now --- .infra/Freyfile.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index 4b4031e..c18ec47 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -39,7 +39,7 @@ [infra.resource.aws_instance.tusd] ami = "${lookup(var.ami, var.region)}" instance_type = "c3.large" - key_name = "tusd" + key_name = "infra-tusd" security_groups = ["fw-tusd-main"] [infra.resource.aws_instance.tusd.connection] key_file = "{{{config.global.ssh.privatekey_file}}}" @@ -168,7 +168,7 @@ ansistrano_group = "ubuntu" [[deploy.playbooks.tasks]] - copy = "src=../env.sh dest={{{config.global.approot}}}/current/env.sh mode=0600 owner=root group=root" + copy = "src=../env.sh dest={{{config.global.approot}}}/current/tusd_linux_amd64/env.sh mode=0600 owner=root group=root" name = "tusd | Upload environment" [[deploy.playbooks.tasks]] From 536727e92d83ff044ee8f943d02f5e4721d1e1b0 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 16:43:19 +0200 Subject: [PATCH 16/25] Save state --- .infra/Frey-state-terraform.tfstate | 36 +++++++++++----------- .infra/Frey-state-terraform.tfstate.backup | 36 +++++++++++----------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.infra/Frey-state-terraform.tfstate b/.infra/Frey-state-terraform.tfstate index a711876..2904501 100644 --- a/.infra/Frey-state-terraform.tfstate +++ b/.infra/Frey-state-terraform.tfstate @@ -1,6 +1,6 @@ { "version": 1, - "serial": 20, + "serial": 24, "modules": [ { "path": [ @@ -8,37 +8,37 @@ ], "outputs": { "endpoint": "http://master.tus.io:80/", - "public_address": "ec2-54-161-247-36.compute-1.amazonaws.com", - "public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com" + "public_address": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_addresses": "ec2-54-166-147-2.compute-1.amazonaws.com" }, "resources": { - "aws_instance.infra-tusd-server": { + "aws_instance.tusd": { "type": "aws_instance", "primary": { - "id": "i-d59cff7e", + "id": "i-16ef798d", "attributes": { "ami": "ami-9bce7af0", - "availability_zone": "us-east-1e", + "availability_zone": "us-east-1d", "ebs_block_device.#": "0", "ebs_optimized": "false", "ephemeral_block_device.#": "0", "iam_instance_profile": "", - "id": "i-d59cff7e", + "id": "i-16ef798d", "instance_state": "running", "instance_type": "c3.large", "key_name": "infra-tusd", "monitoring": "false", - "private_dns": "ip-10-167-178-105.ec2.internal", - "private_ip": "10.167.178.105", - "public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com", - "public_ip": "54.161.247.36", + "private_dns": "ip-10-99-128-27.ec2.internal", + "private_ip": "10.99.128.27", + "public_dns": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_ip": "54.166.147.2", "root_block_device.#": "1", "root_block_device.0.delete_on_termination": "true", "root_block_device.0.iops": "24", "root_block_device.0.volume_size": "8", "root_block_device.0.volume_type": "gp2", "security_groups.#": "1", - "security_groups.1246499019": "fw-infra-tusd-main", + "security_groups.1898427277": "fw-tusd-main", "source_dest_check": "true", "subnet_id": "", "tags.#": "1", @@ -54,7 +54,7 @@ "aws_route53_record.www": { "type": "aws_route53_record", "depends_on": [ - "aws_instance.infra-tusd-server" + "aws_instance.tusd" ], "primary": { "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", @@ -65,7 +65,7 @@ "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", "name": "master.tus.io", "records.#": "1", - "records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com", + "records.3871998453": "ec2-54-166-147-2.compute-1.amazonaws.com", "set_identifier": "", "ttl": "300", "type": "CNAME", @@ -74,14 +74,14 @@ } } }, - "aws_security_group.fw-infra-tusd-main": { + "aws_security_group.fw-tusd-main": { "type": "aws_security_group", "primary": { - "id": "sg-2ff78c42", + "id": "sg-60cc290d", "attributes": { "description": "Infra tusd", "egress.#": "0", - "id": "sg-2ff78c42", + "id": "sg-60cc290d", "ingress.#": "3", "ingress.2214680975.cidr_blocks.#": "1", "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", @@ -106,7 +106,7 @@ "ingress.614077637.security_groups.#": "0", "ingress.614077637.self": "false", "ingress.614077637.to_port": "22", - "name": "fw-infra-tusd-main", + "name": "fw-tusd-main", "owner_id": "402421253186", "tags.#": "0", "vpc_id": "" diff --git a/.infra/Frey-state-terraform.tfstate.backup b/.infra/Frey-state-terraform.tfstate.backup index a711876..2904501 100644 --- a/.infra/Frey-state-terraform.tfstate.backup +++ b/.infra/Frey-state-terraform.tfstate.backup @@ -1,6 +1,6 @@ { "version": 1, - "serial": 20, + "serial": 24, "modules": [ { "path": [ @@ -8,37 +8,37 @@ ], "outputs": { "endpoint": "http://master.tus.io:80/", - "public_address": "ec2-54-161-247-36.compute-1.amazonaws.com", - "public_addresses": "ec2-54-161-247-36.compute-1.amazonaws.com" + "public_address": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_addresses": "ec2-54-166-147-2.compute-1.amazonaws.com" }, "resources": { - "aws_instance.infra-tusd-server": { + "aws_instance.tusd": { "type": "aws_instance", "primary": { - "id": "i-d59cff7e", + "id": "i-16ef798d", "attributes": { "ami": "ami-9bce7af0", - "availability_zone": "us-east-1e", + "availability_zone": "us-east-1d", "ebs_block_device.#": "0", "ebs_optimized": "false", "ephemeral_block_device.#": "0", "iam_instance_profile": "", - "id": "i-d59cff7e", + "id": "i-16ef798d", "instance_state": "running", "instance_type": "c3.large", "key_name": "infra-tusd", "monitoring": "false", - "private_dns": "ip-10-167-178-105.ec2.internal", - "private_ip": "10.167.178.105", - "public_dns": "ec2-54-161-247-36.compute-1.amazonaws.com", - "public_ip": "54.161.247.36", + "private_dns": "ip-10-99-128-27.ec2.internal", + "private_ip": "10.99.128.27", + "public_dns": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_ip": "54.166.147.2", "root_block_device.#": "1", "root_block_device.0.delete_on_termination": "true", "root_block_device.0.iops": "24", "root_block_device.0.volume_size": "8", "root_block_device.0.volume_type": "gp2", "security_groups.#": "1", - "security_groups.1246499019": "fw-infra-tusd-main", + "security_groups.1898427277": "fw-tusd-main", "source_dest_check": "true", "subnet_id": "", "tags.#": "1", @@ -54,7 +54,7 @@ "aws_route53_record.www": { "type": "aws_route53_record", "depends_on": [ - "aws_instance.infra-tusd-server" + "aws_instance.tusd" ], "primary": { "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", @@ -65,7 +65,7 @@ "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", "name": "master.tus.io", "records.#": "1", - "records.1116730303": "ec2-54-161-247-36.compute-1.amazonaws.com", + "records.3871998453": "ec2-54-166-147-2.compute-1.amazonaws.com", "set_identifier": "", "ttl": "300", "type": "CNAME", @@ -74,14 +74,14 @@ } } }, - "aws_security_group.fw-infra-tusd-main": { + "aws_security_group.fw-tusd-main": { "type": "aws_security_group", "primary": { - "id": "sg-2ff78c42", + "id": "sg-60cc290d", "attributes": { "description": "Infra tusd", "egress.#": "0", - "id": "sg-2ff78c42", + "id": "sg-60cc290d", "ingress.#": "3", "ingress.2214680975.cidr_blocks.#": "1", "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", @@ -106,7 +106,7 @@ "ingress.614077637.security_groups.#": "0", "ingress.614077637.self": "false", "ingress.614077637.to_port": "22", - "name": "fw-infra-tusd-main", + "name": "fw-tusd-main", "owner_id": "402421253186", "tags.#": "0", "vpc_id": "" From 8ed24a65d99495624baf7e7d2ec106664a3c03a5 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 17:01:07 +0200 Subject: [PATCH 17/25] Run on legacy build infra for now until Frey supports sudo-less pip installs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 72da320..c322da5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ go: - 1.5 - 1.6rc2 - tip -sudo: false +sudo: required cache: apt: true directories: From 4879004fe4ab45f9b4b41916a9cf42b3a7542c35 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 17:22:09 +0200 Subject: [PATCH 18/25] Open up port 22 so Travis CI can deploy to it --- .infra/Freyfile.toml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index c18ec47..c85e2af 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -18,14 +18,6 @@ [infra.variable.ami.default] us-east-1 = "ami-9bce7af0" -[infra.variable.ip_all] - default = "0.0.0.0/0" -[infra.variable.ip_kevin] - default = "62.163.187.106/32" -[infra.variable.ip_marius] - default = "84.146.0.0/16" -[infra.variable.ip_tim] - default = "24.134.75.132/32" [infra.variable.region] default = "us-east-1" @@ -58,21 +50,17 @@ description = "Infra tusd" name = "fw-tusd-main" [[infra.resource.aws_security_group.fw-tusd-main.ingress]] - cidr_blocks = ["${var.ip_all}"] + cidr_blocks = [ "0.0.0.0/0" ] from_port = 8080 protocol = "tcp" to_port = 8080 [[infra.resource.aws_security_group.fw-tusd-main.ingress]] - cidr_blocks = ["${var.ip_all}"] + cidr_blocks = [ "0.0.0.0/0" ] from_port = 80 protocol = "tcp" to_port = 80 [[infra.resource.aws_security_group.fw-tusd-main.ingress]] - cidr_blocks = [ - "${var.ip_kevin}", - "${var.ip_marius}", - "${var.ip_tim}", - ] + cidr_blocks = [ "0.0.0.0/0" ] from_port = 22 protocol = "tcp" to_port = 22 From 2205a84499913ffca11b234917a10b3a350d584e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 17:22:20 +0200 Subject: [PATCH 19/25] Save state --- .infra/Frey-state-terraform.tfstate | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.infra/Frey-state-terraform.tfstate b/.infra/Frey-state-terraform.tfstate index 2904501..5a2828e 100644 --- a/.infra/Frey-state-terraform.tfstate +++ b/.infra/Frey-state-terraform.tfstate @@ -1,6 +1,6 @@ { "version": 1, - "serial": 24, + "serial": 25, "modules": [ { "path": [ @@ -90,6 +90,13 @@ "ingress.2214680975.security_groups.#": "0", "ingress.2214680975.self": "false", "ingress.2214680975.to_port": "80", + "ingress.2541437006.cidr_blocks.#": "1", + "ingress.2541437006.cidr_blocks.0": "0.0.0.0/0", + "ingress.2541437006.from_port": "22", + "ingress.2541437006.protocol": "tcp", + "ingress.2541437006.security_groups.#": "0", + "ingress.2541437006.self": "false", + "ingress.2541437006.to_port": "22", "ingress.516175195.cidr_blocks.#": "1", "ingress.516175195.cidr_blocks.0": "0.0.0.0/0", "ingress.516175195.from_port": "8080", @@ -97,15 +104,6 @@ "ingress.516175195.security_groups.#": "0", "ingress.516175195.self": "false", "ingress.516175195.to_port": "8080", - "ingress.614077637.cidr_blocks.#": "3", - "ingress.614077637.cidr_blocks.0": "62.163.187.106/32", - "ingress.614077637.cidr_blocks.1": "84.146.0.0/16", - "ingress.614077637.cidr_blocks.2": "24.134.75.132/32", - "ingress.614077637.from_port": "22", - "ingress.614077637.protocol": "tcp", - "ingress.614077637.security_groups.#": "0", - "ingress.614077637.self": "false", - "ingress.614077637.to_port": "22", "name": "fw-tusd-main", "owner_id": "402421253186", "tags.#": "0", From f0f16865a9c64dc0fe9350e5f03cb34e1b8fb33e Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 17:52:46 +0200 Subject: [PATCH 20/25] Rebuild env so it can be deployed to the tus server --- .infra/Freyfile.toml | 2 +- .infra/scripts/encrypt.sh | 37 +++++++++++++++++++++++++++ .infra/scripts/rebuild-env.sh | 47 +++++++++++++++++++++++++++++++++++ .travis.yml | 1 + 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 .infra/scripts/encrypt.sh create mode 100755 .infra/scripts/rebuild-env.sh diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index c85e2af..073ceee 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -156,7 +156,7 @@ ansistrano_group = "ubuntu" [[deploy.playbooks.tasks]] - copy = "src=../env.sh dest={{{config.global.approot}}}/current/tusd_linux_amd64/env.sh mode=0600 owner=root group=root" + copy = "src={{ playbook_dir }}/env.sh dest={{{config.global.approot}}}/current/tusd_linux_amd64/env.sh mode=0600 owner=root group=root" name = "tusd | Upload environment" [[deploy.playbooks.tasks]] diff --git a/.infra/scripts/encrypt.sh b/.infra/scripts/encrypt.sh new file mode 100755 index 0000000..7289bb4 --- /dev/null +++ b/.infra/scripts/encrypt.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Uppy-server. Copyright (c) 2016, Transloadit Ltd. +# +# This file: +# +# - Walks over any FREY_ and TUSD_ environment variable (except for _AWS_) +# - Adds encrypted keys ready for use to .travis.yml +# +# Run as: +# +# source env.sh && ./encrypt.sh +# +# Authors: +# +# - Kevin van Zonneveld + +set -o pipefail +set -o errexit +set -o nounset +# set -o xtrace + +if [ -z "${FREY_DOMAIN:-}" ]; then + echo "FREY_DOMAIN not present. " + echo "Please first source env.sh" + exit 1 +fi + +# Set magic variables for current file & dir +__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +__file="${__dir}/$(basename "${BASH_SOURCE[0]}")" +__base="$(basename ${__file} .sh)" +__freyroot="$(dirname "${__dir}")" + +for var in $(env |awk -F= '{print $1}' |egrep '^(FREY|TUSD)_[A-Z0-9_]+$' |grep -v '_AWS_' |sort); do + echo "Encrypting and adding '${var}'" + travis encrypt "${var}=${!var}" --add env.global +done diff --git a/.infra/scripts/rebuild-env.sh b/.infra/scripts/rebuild-env.sh new file mode 100755 index 0000000..577f232 --- /dev/null +++ b/.infra/scripts/rebuild-env.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# tusd. Copyright (c) 2016, Transloadit Ltd. + +# This file: +# +# - Creates a brand new env.sh based on env.example.sh that is shipped with Git +# - Walks over any FREY_ and TUSD_ environment variable +# - Adds them as exports to to env.sh inside $__freyroot +# +# Run as: +# +# ./rebuild-env.sh +# +# Authors: +# +# - Kevin van Zonneveld + +set -o pipefail +set -o errexit +set -o nounset +# set -o xtrace + +if [ -z "${FREY_DOMAIN:-}" ]; then + echo "FREY_DOMAIN not present. " + echo "Please make sure your environment is set properly (via e.g. travis encrypt)" + exit 1 +fi + +# Set magic variables for current file & dir +__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +__file="${__dir}/$(basename "${BASH_SOURCE[0]}")" +__base="$(basename ${__file} .sh)" +__freyroot="$(dirname "${__dir}")" + +if [ -f "${__freyroot}/env.sh" ]; then + echo "You alreayd have a '${__freyroot}/env.sh'" + exit 1 +fi + +cp -v "${__freyroot}/env.example.sh" "${__freyroot}/env.sh" +chmod 600 "${__freyroot}/env.sh" +for var in $(env |awk -F= '{print $1}' |egrep '^(FREY|TUSD)_[A-Z0-9_]+$'| grep -v '_AWS_' |sort); do + echo "Adding '${var}' to env.sh" + echo "export ${var}=\"${!var}\"" >> "${__freyroot}/env.sh" +done + +ls -al "${__freyroot}/env.sh" diff --git a/.travis.yml b/.travis.yml index c322da5..b8fccc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ before_deploy: - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" - goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" +- .infra/scripts/rebuild-env.sh - if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --force-yes --projectDir .infra); else echo "Skipping deploy fornon-master/PRs"; fi deploy: provider: releases From 603fce755938d88fb61a67ed51bb137b5bf2552f Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Tue, 29 Mar 2016 19:41:44 +0200 Subject: [PATCH 21/25] Now that Frey deploys via Travis are working, only do it on tags, and after release --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8fccc2..c92a4b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,6 @@ before_deploy: - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" - goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" -- .infra/scripts/rebuild-env.sh -- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --force-yes --projectDir .infra); else echo "Skipping deploy fornon-master/PRs"; fi deploy: provider: releases api_key: @@ -48,6 +46,9 @@ deploy: - snapshot/tusd_windows_amd64.zip skip_cleanup: true on: - # tags: true + tags: true go: 1.5 repo: tus/tusd +after_deploy: +- .infra/scripts/rebuild-env.sh +- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --force-yes --projectDir .infra); else echo "Skipping deploy for non-master/PRs"; fi From 8e91ba9b92f8d25035a83a5235ad59ea30847957 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Wed, 30 Mar 2016 16:13:20 +0200 Subject: [PATCH 22/25] We no longer need any env.sh --- .infra/Freyfile.toml | 8 ++---- .infra/scripts/encrypt.sh | 37 --------------------------- .infra/scripts/rebuild-env.sh | 47 ----------------------------------- Makefile | 6 ++++- 4 files changed, 7 insertions(+), 91 deletions(-) delete mode 100755 .infra/scripts/encrypt.sh delete mode 100755 .infra/scripts/rebuild-env.sh diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index 073ceee..afa9cc5 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -122,14 +122,14 @@ upstart_pidfile_path = "{{{config.global.approot}}}/shared/{{{config.global.appname}}}.pid" upstart_respawn = true upstart_respawn_limit = true - upstart_runtime_root = "{{{config.global.approot}}}/current/tusd_linux_amd64/" + upstart_runtime_root = "{{{config.global.approot}}}/current/tusd_linux_amd64" upstart_user = "www-data" [[setup.playbooks.roles]] role = "{{{init.paths.roles_dir}}}/rsyslog/v3.0.1" [setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd] # Send {{{config.global.appname}}} messages to a dedicated logfile - directives = [ "&~" ] + directives = [ "& stop" ] [[setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd.rules]] rule = ":programname, startswith, \"{{{config.global.appname}}}\"" logpath = "{{{config.global.approot}}}/shared/logs/{{{config.global.appname}}}.log" @@ -155,10 +155,6 @@ ansistrano_deploy_via = "download_unarchive" ansistrano_group = "ubuntu" - [[deploy.playbooks.tasks]] - copy = "src={{ playbook_dir }}/env.sh dest={{{config.global.approot}}}/current/tusd_linux_amd64/env.sh mode=0600 owner=root group=root" - name = "tusd | Upload environment" - [[deploy.playbooks.tasks]] name = "tusd | Set file attributes" file = "path={{{config.global.approot}}}/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" diff --git a/.infra/scripts/encrypt.sh b/.infra/scripts/encrypt.sh deleted file mode 100755 index 7289bb4..0000000 --- a/.infra/scripts/encrypt.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Uppy-server. Copyright (c) 2016, Transloadit Ltd. -# -# This file: -# -# - Walks over any FREY_ and TUSD_ environment variable (except for _AWS_) -# - Adds encrypted keys ready for use to .travis.yml -# -# Run as: -# -# source env.sh && ./encrypt.sh -# -# Authors: -# -# - Kevin van Zonneveld - -set -o pipefail -set -o errexit -set -o nounset -# set -o xtrace - -if [ -z "${FREY_DOMAIN:-}" ]; then - echo "FREY_DOMAIN not present. " - echo "Please first source env.sh" - exit 1 -fi - -# Set magic variables for current file & dir -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -__file="${__dir}/$(basename "${BASH_SOURCE[0]}")" -__base="$(basename ${__file} .sh)" -__freyroot="$(dirname "${__dir}")" - -for var in $(env |awk -F= '{print $1}' |egrep '^(FREY|TUSD)_[A-Z0-9_]+$' |grep -v '_AWS_' |sort); do - echo "Encrypting and adding '${var}'" - travis encrypt "${var}=${!var}" --add env.global -done diff --git a/.infra/scripts/rebuild-env.sh b/.infra/scripts/rebuild-env.sh deleted file mode 100755 index 577f232..0000000 --- a/.infra/scripts/rebuild-env.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# tusd. Copyright (c) 2016, Transloadit Ltd. - -# This file: -# -# - Creates a brand new env.sh based on env.example.sh that is shipped with Git -# - Walks over any FREY_ and TUSD_ environment variable -# - Adds them as exports to to env.sh inside $__freyroot -# -# Run as: -# -# ./rebuild-env.sh -# -# Authors: -# -# - Kevin van Zonneveld - -set -o pipefail -set -o errexit -set -o nounset -# set -o xtrace - -if [ -z "${FREY_DOMAIN:-}" ]; then - echo "FREY_DOMAIN not present. " - echo "Please make sure your environment is set properly (via e.g. travis encrypt)" - exit 1 -fi - -# Set magic variables for current file & dir -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -__file="${__dir}/$(basename "${BASH_SOURCE[0]}")" -__base="$(basename ${__file} .sh)" -__freyroot="$(dirname "${__dir}")" - -if [ -f "${__freyroot}/env.sh" ]; then - echo "You alreayd have a '${__freyroot}/env.sh'" - exit 1 -fi - -cp -v "${__freyroot}/env.example.sh" "${__freyroot}/env.sh" -chmod 600 "${__freyroot}/env.sh" -for var in $(env |awk -F= '{print $1}' |egrep '^(FREY|TUSD)_[A-Z0-9_]+$'| grep -v '_AWS_' |sort); do - echo "Adding '${var}' to env.sh" - echo "export ${var}=\"${!var}\"" >> "${__freyroot}/env.sh" -done - -ls -al "${__freyroot}/env.sh" diff --git a/Makefile b/Makefile index 7e5c818..583d620 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PHONY: frey frey: - @npm install --global frey@0.3.12 + @npm install --global frey@0.3.13 PHONY: deploy deploy: @@ -17,3 +17,7 @@ console: PHONY: deploy-localfrey deploy-localfrey: @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js setup + +PHONY: console-localfrey +console-localfrey: + @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js remote From 6cdcb43713cd6ea1c799b57b73ccaa586e7ea542 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Thu, 31 Mar 2016 12:25:59 +0200 Subject: [PATCH 23/25] Configurable Frey version --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 583d620..5eb3e4b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,12 @@ +FREY_VERSION := 0.3.13 + PHONY: frey frey: - @npm install --global frey@0.3.13 + @grep $(FREY_VERSION) node_modules/frey/package.json 2>&1 > /dev/null || npm install frey@$(FREY_VERSION) + +PHONY: provision +provision: frey + @source env.sh && node_modules/.bin/frey install PHONY: deploy deploy: From 58912fdcdac335ee72e8c56a1d943c6fb5da17c4 Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Thu, 31 Mar 2016 12:27:16 +0200 Subject: [PATCH 24/25] .PHONY --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 5eb3e4b..0b86608 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,29 @@ FREY_VERSION := 0.3.13 -PHONY: frey +.PHONY: frey frey: @grep $(FREY_VERSION) node_modules/frey/package.json 2>&1 > /dev/null || npm install frey@$(FREY_VERSION) -PHONY: provision +.PHONY: provision provision: frey @source env.sh && node_modules/.bin/frey install -PHONY: deploy +.PHONY: deploy deploy: @cd .infra && source env.sh && frey setup -PHONY: launch +.PHONY: launch launch: @cd .infra && source env.infra.sh && frey infra -PHONY: console +.PHONY: console console: @cd .infra && source env.sh && frey remote -PHONY: deploy-localfrey +.PHONY: deploy-localfrey deploy-localfrey: @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js setup -PHONY: console-localfrey +.PHONY: console-localfrey console-localfrey: @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js remote From de682b2fe61ef7ffc83036f4ee66152e973732af Mon Sep 17 00:00:00 2001 From: Kevin van Zonneveld Date: Thu, 31 Mar 2016 15:38:16 +0200 Subject: [PATCH 25/25] unattended-upgrades --- .infra/Freyfile.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml index afa9cc5..248cca5 100644 --- a/.infra/Freyfile.toml +++ b/.infra/Freyfile.toml @@ -88,6 +88,9 @@ "wget", ] + [[install.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/unattended-upgrades/v1.2.0" + [[install.playbooks.tasks]] lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" name = "Common | Add convenience shortcut wtf"