wip
This commit is contained in:
parent
ab5ca1717b
commit
046e7995ab
GPG key ID:
4B9DB28F826553BD
5 changed files with 370 additions and 7 deletions
78
artifacts.go
Normal file
78
artifacts.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package artifacts
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// run, job, err
|
||||
func JobInfo() (string, string, error) {
|
||||
rt := os.Getenv("ACTIONS_RUNTIME_TOKEN")
|
||||
var empty string
|
||||
if rt == "" {
|
||||
return empty, empty, fmt.Errorf("env ACTIONS_RUNTIME_TOKEN is only available from the context of an action")
|
||||
}
|
||||
jt, err := newJwtDecode(rt)
|
||||
if err != nil {
|
||||
return empty, empty, err
|
||||
}
|
||||
scp, ok := jt.Payload["scp"].(string)
|
||||
if !ok {
|
||||
return empty, empty, fmt.Errorf("no scp in ACTIONS_RUNTIME_TOKEN")
|
||||
}
|
||||
for _, scopes := range strings.Split(scp, " ") {
|
||||
parts := strings.Split(scopes, ":")
|
||||
if parts[0] != "Actions.Results" {
|
||||
continue
|
||||
}
|
||||
if len(parts) != 3 {
|
||||
return empty, empty, fmt.Errorf("invalid scp in ACTIONS_RUNTIME_TOKEN")
|
||||
}
|
||||
return parts[1], parts[2], nil
|
||||
}
|
||||
|
||||
return empty, empty, fmt.Errorf("no backend ids found")
|
||||
}
|
||||
|
||||
type jwt struct {
|
||||
Header map[string]interface{}
|
||||
Payload map[string]interface{}
|
||||
Signature string
|
||||
}
|
||||
|
||||
func newJwtDecode(token string) (*jwt, error) {
|
||||
parts := strings.Split(token, ".")
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("invalid JWT: expected 3 segments")
|
||||
}
|
||||
|
||||
decode := func(seg string) ([]byte, error) {
|
||||
if l := len(seg) % 4; l != 0 {
|
||||
seg += strings.Repeat("=", 4-l)
|
||||
}
|
||||
return base64.URLEncoding.DecodeString(seg)
|
||||
}
|
||||
|
||||
headerBytes, err := decode(parts[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid JWT header: %v", err)
|
||||
}
|
||||
var header map[string]interface{}
|
||||
if err := json.Unmarshal(headerBytes, &header); err != nil {
|
||||
return nil, fmt.Errorf("invalid JWT header json: %v", err)
|
||||
}
|
||||
|
||||
payloadBytes, err := decode(parts[1])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid JWT payload: %v", err)
|
||||
}
|
||||
var payload map[string]interface{}
|
||||
if err := json.Unmarshal(payloadBytes, &payload); err != nil {
|
||||
return nil, fmt.Errorf("invalid JWT payload json: %v", err)
|
||||
}
|
||||
|
||||
return &jwt{Header: header, Payload: payload, Signature: parts[2]}, nil
|
||||
}
|
||||
188
client.go
Normal file
188
client.go
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ActionClient struct {
|
||||
*http.Client
|
||||
base string
|
||||
token string
|
||||
}
|
||||
|
||||
func NewActionClient() *ActionClient {
|
||||
c := &ActionClient{Client: &http.Client{}}
|
||||
c.base = os.Getenv("ACTIONS_RESULTS_URL")
|
||||
c.token = fmt.Sprintf("Bearer %s", os.Getenv("ACTIONS_RUNTIME_TOKEN"))
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ActionClient) Do(req *http.Request) (*http.Response, error) {
|
||||
req.Header.Set("Authorization", c.token)
|
||||
if !req.URL.IsAbs() {
|
||||
u, err := url.Parse(fmt.Sprintf("%s%s", strings.TrimRight(c.base, "/"), req.URL))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.URL = u
|
||||
}
|
||||
return c.Client.Do(req)
|
||||
}
|
||||
|
||||
func HasContentType(r *http.Response, mimetype string) bool {
|
||||
contentType := r.Header.Get("content-type")
|
||||
if contentType == "" {
|
||||
return mimetype == "application/octet-stream"
|
||||
}
|
||||
|
||||
for _, v := range strings.Split(contentType, ",") {
|
||||
t, _, err := mime.ParseMediaType(v)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if t == mimetype {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type ApiClient interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
type GitHub struct {
|
||||
client ApiClient
|
||||
}
|
||||
|
||||
func NewGitHub(client ApiClient) *GitHub {
|
||||
return &GitHub{client: client}
|
||||
}
|
||||
|
||||
type CreateArtifactRequest struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
Name string `json:"name"`
|
||||
Version int32 `json:"version"`
|
||||
|
||||
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type CreateArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
SignedUploadUrl string `json:"signed_upload_url"`
|
||||
}
|
||||
|
||||
type FinalizeArtifactRequest struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Hash string `json:"hash,omitempty"`
|
||||
}
|
||||
|
||||
type FinalizeArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
ArtifactId string `json:"artifactId"`
|
||||
}
|
||||
|
||||
type ListArtifactsRequest struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
NameFilter string `json:nameFilter`
|
||||
IdFilter string `json:idFilter`
|
||||
}
|
||||
|
||||
type ListArtifactsResponse struct {
|
||||
Artifacts []struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
Id string `json:"databaseId"`
|
||||
Name string `json:"name"`
|
||||
Size string `json:"size"`
|
||||
CreatedAt *time.Time `json:"createdAt"`
|
||||
Digest string `json:"digest"`
|
||||
} `json:artifacts`
|
||||
}
|
||||
|
||||
type GetSignedArtifactURLRequest struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GetSignedArtifactURLResponse struct {
|
||||
SignedUrl string `json:"signedUrl"`
|
||||
}
|
||||
|
||||
type DeleteArtifactRequest struct {
|
||||
RunID string `json:"workflowRunBackendId"`
|
||||
JobRunID string `json:"workflowJobRunBackendId"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type DeleteArtifactResponse struct {
|
||||
Ok bool `json:"ok"`
|
||||
ArtifactId string `json:"artifactId"`
|
||||
}
|
||||
|
||||
func (gh *GitHub) CreateArtifact(ctx context.Context, req CreateArtifactRequest) (*CreateArtifactResponse, error) {
|
||||
endpoint := "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact"
|
||||
var result CreateArtifactResponse
|
||||
if err := gh.twirp(ctx, endpoint, req, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (gh *GitHub) FinalizeArtifact(ctx context.Context, req FinalizeArtifactRequest) (*FinalizeArtifactResponse, error) {
|
||||
endpoint := "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact"
|
||||
var result FinalizeArtifactResponse
|
||||
if err := gh.twirp(ctx, endpoint, req, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (gh *GitHub) twirp(ctx context.Context, url string, in any, out any) error {
|
||||
var b bytes.Buffer
|
||||
|
||||
if err := json.NewEncoder(&b).Encode(in); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("content-type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
res, err := gh.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if !HasContentType(res, "application/json") {
|
||||
return fmt.Errorf("content-type: %q should be %q",
|
||||
res.Header.Get("content-type"), "application/json")
|
||||
}
|
||||
dec := json.NewDecoder(io.LimitReader(res.Body, 32*1024))
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
return dec.Decode(out)
|
||||
} else {
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
return fmt.Errorf("twirp call %s failed: %d: %s", url, res.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
28
load/main.go
28
load/main.go
|
|
@ -1,4 +1,32 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"code.geekeey.de/actions/sdk"
|
||||
)
|
||||
|
||||
func main() {
|
||||
action := &SaveArtifactAction{Action: sdk.New()}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer cancel()
|
||||
if err := action.Run(ctx); err != nil {
|
||||
action.Errorf("%s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type SaveArtifactAction struct {
|
||||
*sdk.Action
|
||||
name string
|
||||
}
|
||||
|
||||
func (action *SaveArtifactAction) setup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (action *SaveArtifactAction) Run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
65
save/main.go
65
save/main.go
|
|
@ -1,11 +1,18 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"code.geekeey.de/actions/artifacts"
|
||||
"code.geekeey.de/actions/sdk"
|
||||
"code.geekeey.de/actions/sdk/iox"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
|
@ -20,9 +27,7 @@ func main() {
|
|||
|
||||
type SaveArtifactAction struct {
|
||||
*sdk.Action
|
||||
repository string
|
||||
path string
|
||||
reference string
|
||||
name string
|
||||
}
|
||||
|
||||
func (action *SaveArtifactAction) setup() error {
|
||||
|
|
@ -33,5 +38,59 @@ func (action *SaveArtifactAction) Run(ctx context.Context) error {
|
|||
if err := action.setup(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
run, job, err := artifacts.JobInfo()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get job info: %v", err)
|
||||
}
|
||||
|
||||
github := artifacts.NewGitHub(artifacts.NewActionClient())
|
||||
|
||||
expire := time.Now().Add(20 * time.Hour)
|
||||
create, err := github.CreateArtifact(ctx, artifacts.CreateArtifactRequest{
|
||||
RunID: run,
|
||||
JobRunID: job,
|
||||
Name: action.name,
|
||||
Version: 4,
|
||||
ExpiresAt: &expire,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create artifact %w", err)
|
||||
}
|
||||
if !create.Ok {
|
||||
return fmt.Errorf("cannot get pre-signed URL")
|
||||
}
|
||||
|
||||
rd, err := action.CreateArtifact(ctx)
|
||||
|
||||
res, err := artifacts.PushBlob_(ctx, rd, create.SignedUploadUrl, 1024*1024*1, runtime.NumCPU())
|
||||
|
||||
finish, err := github.FinalizeArtifact(ctx, artifacts.FinalizeArtifactRequest{
|
||||
RunID: run,
|
||||
JobRunID: job,
|
||||
Name: action.name,
|
||||
Size: res.Size,
|
||||
Hash: res.SHA256Sum,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot finish artifact %w", err)
|
||||
}
|
||||
if !create.Ok {
|
||||
return fmt.Errorf("cannot finish artifact upload")
|
||||
}
|
||||
|
||||
action.Noticef("created artifact: %s", finish.ArtifactId)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (action *SaveArtifactAction) CreateArtifact(ctx context.Context) (io.Reader, error) {
|
||||
rd, wr, err := os.Pipe()
|
||||
go func() {
|
||||
defer wr.Close()
|
||||
|
||||
t := tar.NewWriter(wr)
|
||||
t.AddFS(iox.NewGlobFS(os.DirFS(""), ""))
|
||||
}()
|
||||
return rd, err
|
||||
}
|
||||
|
|
|
|||
154
storage.go
Normal file
154
storage.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type PullResult struct {
|
||||
SHA256Sum string
|
||||
Size int64
|
||||
}
|
||||
type PushResult struct {
|
||||
SHA256Sum string
|
||||
Size int64
|
||||
}
|
||||
|
||||
func PushBlob(ctx context.Context, url string, data io.Reader, chunkSize, maxParallel int) (int, string, error) {
|
||||
var empty string
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 4 * 1024 * 1024 // 4MB default
|
||||
}
|
||||
if maxParallel <= 0 {
|
||||
maxParallel = 4
|
||||
}
|
||||
|
||||
type chunk struct {
|
||||
index int
|
||||
data []byte
|
||||
}
|
||||
wrkinp := make(chan chunk)
|
||||
wrkerr := make(chan error, maxParallel)
|
||||
blockIDs := make([]string, 0)
|
||||
var blockMu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Worker pool for parallel uploading
|
||||
for i := 0; i < maxParallel; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for c := range wrkinp {
|
||||
id := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%06d", c.index)))
|
||||
address := fmt.Sprintf("%s&comp=block&blockid=%s", url, id)
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewReader(c.data))
|
||||
if err != nil {
|
||||
wrkerr <- err
|
||||
return
|
||||
}
|
||||
req.ContentLength = int64(len(c.data))
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
wrkerr <- err
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
wrkerr <- fmt.Errorf("chunk %d upload failed: %s body: %s", c.index, resp.Status, string(body))
|
||||
return
|
||||
}
|
||||
|
||||
blockMu.Lock()
|
||||
defer blockMu.Unlock()
|
||||
blockIDs = append(blockIDs, id)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Chunk the data and send to workers
|
||||
index := 0
|
||||
hash := sha256.New()
|
||||
for {
|
||||
buf := make([]byte, chunkSize)
|
||||
n, err := io.ReadFull(data, buf)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
if n > 0 {
|
||||
hash.Write(buf[:n])
|
||||
wrkinp <- chunk{index: index, data: buf[:n]}
|
||||
index++
|
||||
}
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
close(wrkinp)
|
||||
wg.Wait()
|
||||
return empty, err
|
||||
}
|
||||
hash.Write(buf)
|
||||
wrkinp <- chunk{index: index, data: buf}
|
||||
index++
|
||||
}
|
||||
close(wrkinp)
|
||||
wg.Wait()
|
||||
|
||||
select {
|
||||
case err := <-wrkerr:
|
||||
return empty, err
|
||||
default:
|
||||
}
|
||||
|
||||
type BlockList struct {
|
||||
Latest []string `xml:"Latest"`
|
||||
}
|
||||
|
||||
type Latest struct {
|
||||
Value string `xml:",chardata"`
|
||||
}
|
||||
|
||||
// Sort block IDs to ensure correct order
|
||||
blockMu.Lock()
|
||||
defer blockMu.Unlock()
|
||||
if len(blockIDs) == 0 {
|
||||
return empty, fmt.Errorf("no blocks uploaded")
|
||||
}
|
||||
blockList := &BlockList{}
|
||||
for i := 0; i < len(blockIDs); i++ {
|
||||
blockList.Latest = append(blockList.Blocks, xmlBlock{XMLName: xml.Name{Local: "Latest"}, ID: blockIDs[i]})
|
||||
}
|
||||
blockListData, err := xml.Marshal(blockList)
|
||||
if err != nil {
|
||||
return empty, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", fmt.Sprintf("%s&comp=blocklist", url), bytes.NewReader(blockListData))
|
||||
if err != nil {
|
||||
return empty, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/xml")
|
||||
req.ContentLength = int64(len(blockListData))
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return empty, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return empty, fmt.Errorf("blocklist commit failed: %s body: %s", resp.Status, string(body))
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func PushBlob_(ctx context.Context, r io.Reader, blobURL string, chunkSize int64, parallelism int) (*PushResult, error) {
|
||||
}
|
||||
|
||||
func PullBlob_(ctx context.Context, w io.Writer, blobURL string, chunkSize int64, parallelism int) (*PullResult, error) {
|
||||
}
|
||||
Loading…
Reference in a new issue