Gradle tasks for publishing to bintray, jcenter, mavencentral etc. (#25351)

Summary:
Gradle tasks for publishing to bintray and jcenter, mavencentral; snapshot buidls go to oss.sonatype.org

Those gradle changes adds tasks:

bintrayUpload - publishing on bintray, in 'facebook' org
uploadArchives - uploading to maven repos

Gradle tasks are copied from facebook open sourced libraries like https://github.com/facebook/litho, https://github.com/facebookincubator/spectrum

To do the publishing we need to provide somehow (e.g. in ~/.gradle/gradle.properties)
```
signing.keyId=
signing.password=
signing.secretKeyRingFile=

bintrayUsername=
bintrayApiKey=
bintrayGpgPassword=

SONATYPE_NEXUS_USERNAME=
SONATYPE_NEXUS_PASSWORD=
```

android/libs/fbjni is submodule, to be able to add publishing tasks to it (it needs to be published as separate maven dependency) - I created `android/libs/fbjni_local` that has only `build.gradle` with release tasks.

pytorch_android dependency for ':fbjni' changed from implementation -> api as implementation treated as 'private' dependency which is translated to scope=runtime in maven pom file, api works as 'compile'

Testing:
it's already published on bintray with version 0.0.4 and can be used in gradle files as

```
repositories {
    maven {
        url  "https://dl.bintray.com/facebook/maven"
    }
}

dependencies {
    implementation 'com.facebook:pytorch_android:0.0.4'
    implementation 'com.facebook:pytorch_android_torchvision:0.0.4'
}
```

It was published in com.facebook group

I requested sync to jcenter from bintray, that usually takes 2-3 days

Versioning added version suffixes to aar output files and circleCI jobs for android start failing as they expected just pytorch_android.aar pytorch_android_torchvision.aar, without any version

To avoid it - I changed circleCI android jobs to zip *.aar files and publish as single artifact with name artifacts.zip, I will add kostmo to check this part, if circleCI jobs finish ok - everything works :)
Pull Request resolved: https://github.com/pytorch/pytorch/pull/25351

Reviewed By: kostmo

Differential Revision: D17135886

Pulled By: IvanKobzarev

fbshipit-source-id: 64eebac670bbccaaafa1b04eeab15760dd5ecdf9
This commit is contained in:
Ivan Kobzarev
2019-08-30 17:50:54 -07:00
committed by Facebook Github Bot
parent a27fdfd38c
commit 6e4eeb1d17
19 changed files with 420 additions and 41 deletions

View File

@ -1198,19 +1198,15 @@ jobs:
export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
mkdir -p ~/workspace/build_android_aar
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/pytorch_android/build/outputs/aar/pytorch_android.aar ~/workspace/build_android_aar/
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/pytorch_android_torchvision/build/outputs/aar/pytorch_android_torchvision.aar ~/workspace/build_android_aar/
mkdir -p ~/workspace/build_android_artifacts
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_artifacts/
output_image=$docker_image_libtorch_android_x86_32-gradle
docker commit "$id_x86_32" ${output_image}
docker push ${output_image}
- store_artifacts:
path: ~/workspace/build_android_aar/pytorch_android.aar
destination: pytorch_android.aar
- store_artifacts:
path: ~/workspace/build_android_aar/pytorch_android_torchvision.aar
destination: pytorch_android_torchvision.aar
path: ~/workspace/build_android_artifacts/artifacts.tgz
destination: artifacts.tgz
pytorch_android_gradle_build-x86_32:
environment:
@ -1253,20 +1249,15 @@ jobs:
export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
mkdir -p ~/workspace/build_android_aar_x86_32
docker cp $id:/var/lib/jenkins/workspace/android/pytorch_android/build/outputs/aar/pytorch_android.aar ~/workspace/build_android_aar_x86_32/
docker cp $id:/var/lib/jenkins/workspace/android/pytorch_android_torchvision/build/outputs/aar/pytorch_android_torchvision.aar ~/workspace/build_android_aar_x86_32/
mkdir -p ~/workspace/build_android_x86_32_artifacts
docker cp $id:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_x86_32_artifacts/
output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-gradle-x86_32
docker commit "$id" ${output_image}
docker push ${output_image}
- store_artifacts:
path: ~/workspace/build_android_aar_x86_32/pytorch_android.aar
destination: pytorch_android_x86.aar
- store_artifacts:
path: ~/workspace/build_android_aar_x86_32/pytorch_android_torchvision.aar
destination: pytorch_android_torchvision_x86.aar
path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
destination: artifacts.tgz
# update_s3_htmls job
# These jobs create html files for every cpu/cu## folder in s3. The html

View File

@ -78,4 +78,4 @@ else
$GRADLE_PATH -p ~/workspace/android/ assembleRelease
fi
find . -type f -name *aar | xargs ls -lah
find . -type f -name *aar -print | xargs tar cfvz ~/workspace/android/artifacts.tgz

View File

@ -347,19 +347,15 @@
export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
mkdir -p ~/workspace/build_android_aar
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/pytorch_android/build/outputs/aar/pytorch_android.aar ~/workspace/build_android_aar/
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/pytorch_android_torchvision/build/outputs/aar/pytorch_android_torchvision.aar ~/workspace/build_android_aar/
mkdir -p ~/workspace/build_android_artifacts
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_artifacts/
output_image=$docker_image_libtorch_android_x86_32-gradle
docker commit "$id_x86_32" ${output_image}
docker push ${output_image}
- store_artifacts:
path: ~/workspace/build_android_aar/pytorch_android.aar
destination: pytorch_android.aar
- store_artifacts:
path: ~/workspace/build_android_aar/pytorch_android_torchvision.aar
destination: pytorch_android_torchvision.aar
path: ~/workspace/build_android_artifacts/artifacts.tgz
destination: artifacts.tgz
pytorch_android_gradle_build-x86_32:
environment:
@ -402,17 +398,12 @@
export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
mkdir -p ~/workspace/build_android_aar_x86_32
docker cp $id:/var/lib/jenkins/workspace/android/pytorch_android/build/outputs/aar/pytorch_android.aar ~/workspace/build_android_aar_x86_32/
docker cp $id:/var/lib/jenkins/workspace/android/pytorch_android_torchvision/build/outputs/aar/pytorch_android_torchvision.aar ~/workspace/build_android_aar_x86_32/
mkdir -p ~/workspace/build_android_x86_32_artifacts
docker cp $id:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_x86_32_artifacts/
output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-gradle-x86_32
docker commit "$id" ${output_image}
docker push ${output_image}
- store_artifacts:
path: ~/workspace/build_android_aar_x86_32/pytorch_android.aar
destination: pytorch_android_x86.aar
- store_artifacts:
path: ~/workspace/build_android_aar_x86_32/pytorch_android_torchvision.aar
destination: pytorch_android_torchvision_x86.aar
path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
destination: artifacts.tgz

1
.gitignore vendored
View File

@ -9,6 +9,7 @@
## PyTorch
.coverage
.gradle
.hypothesis
.mypy_cache
*/*.pyc

View File

@ -21,6 +21,9 @@ buildscript {
dependencies {
classpath 'com.android.tools.build:gradle:3.3.2'
classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:${GRADLE_BINTRAY_PLUGIN_VERSION}"
classpath "com.github.dcendents:android-maven-gradle-plugin:${ANDROID_MAVEN_GRADLE_PLUGIN_VERSION}"
classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4.9.8"
}
}

View File

@ -1 +1,19 @@
ABI_FILTERS=armeabi-v7a,arm64-v8a,x86,x86_64
VERSION_NAME=0.0.4
GROUP=org.pytorch
MAVEN_GROUP=com.facebook
POM_URL=https://github.com/pytorch/pytorch/tree/master/android
POM_SCM_URL=https://github.com/pytorch/pytorch.git
POM_SCM_CONNECTION=scm:git:https://github.com/pytorch/pytorch
POM_SCM_DEV_CONNECTION=scm:git:git@github.com:pytorch/pytorch.git
POM_LICENSE_NAME=BSD 3-Clause
POM_LICENSE_URL=https://github.com/pytorch/pytorch/blob/master/LICENSE
POM_ISSUES_URL=https://github.com/pytorch/pytorch/issues
POM_LICENSE_DIST=repo
POM_DEVELOPER_ID=facebook
POM_DEVELOPER_NAME=facebook
GRADLE_BINTRAY_PLUGIN_VERSION=1.8.0
GRADLE_VERSIONS_PLUGIN_VERSION=0.15.0
ANDROID_MAVEN_GRADLE_PLUGIN_VERSION=2.1

View File

@ -0,0 +1,32 @@
apply plugin: 'com.github.dcendents.android-maven'
version = VERSION_NAME
group = GROUP
project.archivesBaseName = POM_ARTIFACT_ID
install {
repositories.mavenInstaller {
pom.project {
name POM_NAME
artifactId POM_ARTIFACT_ID
packaging POM_PACKAGING
description POM_DESCRIPTION
url projectUrl
scm {
url scmUrl
connection scmConnection
developerConnection scmDeveloperConnection
}
licenses projectLicenses
developers {
developer {
id developerId
name developerName
}
}
}
}
}

View File

@ -0,0 +1,95 @@
import java.nio.file.Files
import java.nio.file.Paths
import java.io.FileOutputStream
import java.util.zip.ZipFile
// Android tasks for Javadoc and sources.jar generation
afterEvaluate { project ->
if (POM_PACKAGING == 'aar') {
task androidJavadoc(type: Javadoc, dependsOn: assembleDebug) {
source += files(android.sourceSets.main.java.srcDirs)
failOnError false
// This task will try to compile *everything* it finds in the above directory and
// will choke on text files it doesn't understand.
exclude '**/BUCK'
exclude '**/*.md'
}
task androidJavadocJar(type: Jar, dependsOn: androidJavadoc) {
archiveClassifier.set('javadoc')
from androidJavadoc.destinationDir
}
task androidSourcesJar(type: Jar) {
archiveClassifier.set('sources')
from android.sourceSets.main.java.srcDirs
}
android.libraryVariants.all { variant ->
def name = variant.name.capitalize()
task "jar${name}"(type: Jar, dependsOn: variant.javaCompileProvider) {
from variant.javaCompileProvider.get().destinationDir
}
androidJavadoc.doFirst {
classpath += files(android.bootClasspath)
classpath += files(variant.javaCompileProvider.get().classpath.files)
// This is generated by `assembleDebug` and holds the JARs generated by the APT.
classpath += fileTree(dir: "$buildDir/intermediates/bundles/debug/", include: '**/*.jar')
// Process AAR dependencies
def aarDependencies = classpath.filter { it.name.endsWith('.aar') }
classpath -= aarDependencies
aarDependencies.each { aar ->
// Extract classes.jar from the AAR dependency, and add it to the javadoc classpath
def outputPath = "$buildDir/tmp/aarJar/${aar.name.replace('.aar', '.jar')}"
classpath += files(outputPath)
// Use a task so the actual extraction only happens before the javadoc task is run
dependsOn task(name: "extract ${aar.name}").doLast {
extractEntry(aar, 'classes.jar', outputPath)
}
}
}
}
artifacts.add('archives', androidJavadocJar)
artifacts.add('archives', androidSourcesJar)
}
if (POM_PACKAGING == 'jar') {
task javadocJar(type: Jar, dependsOn: javadoc) {
archiveClassifier.set('javadoc')
from javadoc.destinationDir
}
task sourcesJar(type: Jar, dependsOn: classes) {
archiveClassifier.set('sources')
from sourceSets.main.allSource
}
artifacts.add('archives', javadocJar)
artifacts.add('archives', sourcesJar)
}
}
// Utility method to extract only one entry in a zip file
private def extractEntry(archive, entryPath, outputPath) {
if (!archive.exists()) {
throw new GradleException("archive $archive not found")
}
def zip = new ZipFile(archive)
zip.entries().each {
if (it.name == entryPath) {
def path = Paths.get(outputPath)
if (!Files.exists(path)) {
Files.createDirectories(path.getParent())
Files.copy(zip.getInputStream(it), path)
}
}
}
zip.close()
}

View File

@ -0,0 +1,64 @@
apply plugin: 'com.jfrog.bintray'
def getBintrayUsername() {
return project.hasProperty('bintrayUsername') ? property('bintrayUsername') : System.getenv('BINTRAY_USERNAME')
}
def getBintrayApiKey() {
return project.hasProperty('bintrayApiKey') ? property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY')
}
def getBintrayGpgPassword() {
return project.hasProperty('bintrayGpgPassword') ? property('bintrayGpgPassword') : System.getenv('BINTRAY_GPG_PASSWORD')
}
def getMavenCentralUsername() {
return project.hasProperty('mavenCentralUsername') ? property('mavenCentralUsername') : System.getenv('MAVEN_CENTRAL_USERNAME')
}
def getMavenCentralPassword() {
return project.hasProperty('mavenCentralPassword') ? property('mavenCentralPassword') : System.getenv('MAVEN_CENTRAL_PASSWORD')
}
def shouldSyncWithMavenCentral() {
return project.hasProperty('syncWithMavenCentral') ? property('syncWithMavenCentral').toBoolean() : false
}
def dryRunOnly() {
return project.hasProperty('dryRun') ? property('dryRun').toBoolean() : false
}
bintray {
user = getBintrayUsername()
key = getBintrayApiKey()
override = false
configurations = ['archives']
pkg {
repo = bintrayRepo
userOrg = bintrayUserOrg
name = bintrayName
desc = bintrayDescription
websiteUrl = projectUrl
issueTrackerUrl = issuesUrl
vcsUrl = scmUrl
licenses = [ POM_LICENSE_NAME ]
dryRun = dryRunOnly()
override = false
publish = true
publicDownloadNumbers = true
version {
name = versionName
desc = bintrayDescription
gpg {
sign = true
passphrase = getBintrayGpgPassword()
}
mavenCentralSync {
sync = shouldSyncWithMavenCentral()
user = getMavenCentralUsername()
password = getMavenCentralPassword()
close = '1' // If set to 0, you have to manually click release
}
}
}
}

View File

@ -0,0 +1,81 @@
apply plugin: 'signing'
version = VERSION_NAME
group = MAVEN_GROUP
def isReleaseBuild() {
return !VERSION_NAME.contains('SNAPSHOT')
}
def getReleaseRepositoryUrl() {
return hasProperty('RELEASE_REPOSITORY_URL') ? RELEASE_REPOSITORY_URL
: "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
}
def getSnapshotRepositoryUrl() {
return hasProperty('SNAPSHOT_REPOSITORY_URL') ? SNAPSHOT_REPOSITORY_URL
: "https://oss.sonatype.org/content/repositories/snapshots/"
}
def getRepositoryUsername() {
return hasProperty('SONATYPE_NEXUS_USERNAME') ? SONATYPE_NEXUS_USERNAME : ""
}
def getRepositoryPassword() {
return hasProperty('SONATYPE_NEXUS_PASSWORD') ? SONATYPE_NEXUS_PASSWORD : ""
}
afterEvaluate { project ->
uploadArchives {
repositories {
mavenDeployer {
beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) }
pom.groupId = MAVEN_GROUP
pom.artifactId = POM_ARTIFACT_ID
pom.version = VERSION_NAME
repository(url: getReleaseRepositoryUrl()) {
authentication(userName: getRepositoryUsername(), password: getRepositoryPassword())
}
snapshotRepository(url: getSnapshotRepositoryUrl()) {
authentication(userName: getRepositoryUsername(), password: getRepositoryPassword())
}
pom.project {
name POM_NAME
packaging POM_PACKAGING
description POM_DESCRIPTION
url POM_URL
scm {
url POM_SCM_URL
connection POM_SCM_CONNECTION
developerConnection POM_SCM_DEV_CONNECTION
}
licenses {
license {
name POM_LICENSE_NAME
url POM_LICENSE_URL
distribution POM_LICENSE_DIST
}
}
developers {
developer {
id POM_DEVELOPER_ID
name POM_DEVELOPER_NAME
}
}
}
}
}
}
signing {
required { isReleaseBuild() && gradle.taskGraph.hasTask('uploadArchives') }
sign configurations.archives
}
}

View File

@ -0,0 +1,5 @@
apply from: rootProject.file('gradle/android_tasks.gradle')
apply from: rootProject.file('gradle/release_bintray.gradle')
apply from: rootProject.file('gradle/gradle_maven_push.gradle')

View File

@ -0,0 +1,32 @@
ext {
bintrayRepo = 'maven'
bintrayUserOrg = 'facebook'
bintrayName = "${GROUP}:${POM_ARTIFACT_ID}"
bintrayDescription = POM_DESCRIPTION
projectUrl = POM_URL
issuesUrl = POM_ISSUES_URL
scmUrl = POM_SCM_URL
scmConnection = POM_SCM_CONNECTION
scmDeveloperConnection = POM_SCM_DEV_CONNECTION
publishedGroupId = GROUP
libraryName = 'pytorch_android'
artifact = 'pytorch_android'
developerId = POM_DEVELOPER_ID
developerName = POM_DEVELOPER_NAME
versionName = VERSION_NAME
projectLicenses = {
license = {
name = POM_LICENSE_NAME
url = POM_LICENSE_URL
distribution = POM_LICENSE_DIST
}
}
}
apply from: rootProject.file('gradle/android_maven_install.gradle')
apply from: rootProject.file('gradle/bintray.gradle')

View File

@ -0,0 +1,33 @@
apply plugin: 'com.android.library'
apply plugin: 'maven'
android {
compileSdkVersion rootProject.compileSdkVersion
buildToolsVersion rootProject.buildToolsVersion
defaultConfig {
minSdkVersion rootProject.minSdkVersion
targetSdkVersion rootProject.targetSdkVersion
sourceSets {
main {
manifest.srcFile '../fbjni/ApplicationManifest.xml'
java {
srcDir '../fbjni/java'
}
}
}
}
externalNativeBuild {
cmake {
path "../fbjni/CMakeLists.txt"
}
}
}
dependencies {
compileOnly 'com.google.code.findbugs:jsr305:3.0.1'
}
apply from: rootProject.file('gradle/release.gradle')

View File

@ -0,0 +1,4 @@
POM_NAME=pytorch_android_fbjni
POM_DESCRIPTION=pytorch_android_fbjni
POM_ARTIFACT_ID=pytorch_android_fbjni
POM_PACKAGING=aar

View File

@ -1,4 +1,5 @@
apply plugin: 'com.android.library'
apply plugin: 'maven'
android {
compileSdkVersion rootProject.compileSdkVersion
@ -7,8 +8,8 @@ android {
defaultConfig {
minSdkVersion rootProject.minSdkVersion
targetSdkVersion rootProject.targetSdkVersion
versionCode 1
versionName "1.0"
versionCode 0
versionName "0.1"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
ndk {
@ -40,7 +41,7 @@ android {
}
dependencies {
implementation project(':fbjni')
api project(':fbjni')
implementation 'com.android.support:appcompat-v7:28.0.0'
@ -53,3 +54,13 @@ dependencies {
androidTestImplementation 'androidx.test:rules:' + rootProject.rulesVersion
androidTestImplementation 'androidx.test:runner:' + rootProject.runnerVersion
}
apply from: rootProject.file('gradle/release.gradle')
task sourcesJar(type: Jar) {
from android.sourceSets.main.java.srcDirs
archiveClassifier.set('sources')
}
artifacts.add('archives', sourcesJar)

View File

@ -0,0 +1,4 @@
POM_NAME=pytorch_android pytorch android api
POM_DESCRIPTION=pytorch_android pytorch android api
POM_ARTIFACT_ID=pytorch_android
POM_PACKAGING=aar

View File

@ -1,4 +1,5 @@
apply plugin: 'com.android.library'
apply plugin: 'maven'
android {
compileSdkVersion rootProject.compileSdkVersion
@ -8,8 +9,8 @@ android {
defaultConfig {
minSdkVersion rootProject.minSdkVersion
targetSdkVersion rootProject.targetSdkVersion
versionCode 1
versionName "0.0"
versionCode 0
versionName "0.1"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
@ -43,3 +44,12 @@ dependencies {
androidTestImplementation 'androidx.test:rules:' + rootProject.rulesVersion
androidTestImplementation 'androidx.test:runner:' + rootProject.runnerVersion
}
apply from: rootProject.file('gradle/release.gradle')
task sourcesJar(type: Jar) {
from android.sourceSets.main.java.srcDirs
archiveClassifier.set('sources')
}
artifacts.add('archives', sourcesJar)

View File

@ -0,0 +1,4 @@
POM_NAME=pytorch_android_torchvision
POM_DESCRIPTION=pytorch_android_torchvision
POM_ARTIFACT_ID=pytorch_android_torchvision
POM_PACKAGING=aar

View File

@ -1,4 +1,4 @@
include ':app', ':pytorch_android', ':fbjni', ':pytorch_android_torchvision'
project(':fbjni').projectDir = file('libs/fbjni')
project(':fbjni').projectDir = file('libs/fbjni_local')
project(':pytorch_android_torchvision').projectDir = file('pytorch_android_torchvision')