Now Passing

This commit is contained in:
2023-05-23 13:15:36 -04:00
parent dcea1444c3
commit 175d46bdd9
14 changed files with 132 additions and 398 deletions

View File

@@ -1,149 +1,2 @@
with builtins;
# This duplicates the overlay code in nixpkgs/pkgs/top-level/impure.nix (may require periodic resyncing).
#
# It has to be done this way because passing overlays to nixpkgs (required to insert our packages) disables all the
# default overlay mechanisms (required to pickup any user or system overlays).
let
# overlaysDefault - List of overlay functions nixpkgs would normally use if not provided.
#
# This is the order overlays are searched for in accordance to what is done in nixpkgs. If an overlay is found
# at a given level (the leading number) the search is stopped and they are used. If multiple overlays are found
# at a given level then an error is generated.
#
# 1. <nixpkgs-overlays> (i.e., nixpkgs-overlays.nix or nixpkgs-overlays/default.nix in NIX_PATH)
# 2. ~/.config/nixpkgs/overlays.nix
# 2. ~/.config/nixpkgs/overlays
#
# This code is lifted directly from nixpkgs/pkgs/top-level/impure.nix to ensure consistency.
#
overlaysDefault =
let
# try expression default - Replace expression with default on exception.
try = x: def: let res = tryEval x; in if res.success then res.value else def;
# isDir path - Is path a directory (requires access).
isDir = path: pathExists (path + "/.");
# overlaysRetrieve path - Retrieve a list of the overlay functions from path.
# path is file - import the file itself (should give a list of overlay functions)
# path is directory - list of imports of all the *.nix files in the directory (each should give an overlay function)
#
overlaysRetrieve = path:
if isDir path then
let content = readDir path; in
map (n: import (path + ("/" + n)))
(builtins.filter (n: builtins.match ".*\\.nix" n != null || pathExists (path + ("/" + n + "/default.nix")))
(attrNames content))
else
import path;
# pathOverlays - NIX_PATH nixpkgs-overlays file or "" if not found
pathOverlays = try <nixpkgs-overlays> "";
# homeDir - ~/
# homeOverlaysFile - ~/.config/nixpkgs/overlays.nix
# homeOverlaysDir - ~/.config/nixpkgs/overlays
#
homeDir = builtins.getEnv "HOME";
homeOverlaysFile = homeDir + "/.config/nixpkgs/overlays.nix";
homeOverlaysDir = homeDir + "/.config/nixpkgs/overlays";
in
if pathOverlays != "" && pathExists pathOverlays then overlaysRetrieve pathOverlays
else if pathExists homeOverlaysFile && pathExists homeOverlaysDir then
throw ''
Nixpkgs overlays can be specified with ${homeOverlaysFile} or ${homeOverlaysDir}, but not both.
Please remove one of them and try again.
''
else if pathExists homeOverlaysFile then
if isDir homeOverlaysFile then
throw (homeOverlaysFile + " should be a file")
else overlaysRetrieve homeOverlaysFile
else if pathExists homeOverlaysDir then
if !(isDir homeOverlaysDir) then
throw (homeOverlaysDir + " should be a directory")
else overlaysRetrieve homeOverlaysDir
else [];
# overlaysAlways - List of overlay functions provide by this package.
#
# The naming in this function is misleading but consistent with nixpkgs.
#
# self - final package set (stack fully closed) (-> pkgs in all-packages.nix)
# super - prior package set (stack closed up to us)
# res - next package set (stack closed up to and over us) (-> self in all-packages.nix)
#
overlaysAlways =
let
# knot - Feed final and up to and over us overlay results into overlay
#
# This matches what is done in nixpkgs (see pkgs/top-level/stage.nix).
#
knot = path: self: super:
let res = import path res super self; in res;
in
map knot [
./temporary/all-packages.nix
./pkgs/all-packages.nix
];
# nixpkgs - The underlying nixpkg to use.
#
# Get a usable nixpkgs, that is, one with a version that matches ours, or die.
#
nixpkgs =
let
# first check list - Return first element of list that passes check list otherwise aborts.
#
first = check: list:
if list == []
then builtins.throw ''
Unable to locate a suitable nixpkgs directory.
Most likely you want to git clone one into the top of the repo as 'upstream'.
''
else if check ( builtins.head list )
then builtins.head list
else first check ( builtins.tail list );
# okay path - Check if path exist and match our version number and otherwise print a warning.
#
# Version numbers are taken from ./version files.
#
okay =
let
version = builtins.readFile ./.version;
in
path:
( builtins.pathExists (path + /.version)
&& ( builtins.readFile (path + /.version) == version
|| builtins.trace ''
Skipping ${toString path} as not version ${version}.
'' false ) );
# paths - Paths to search for nixpkgs.
#
paths = [
./upstream # 1. build channel or git cloned one directly associated with us takes priority
<nixpkgs/upstream> # 2. if the channel is a version of ourselves the real one will be in a subdirectory
<nixpkgs> # 3. otherwise maybe the channel is a nixpkgs proper
];
in
first okay paths;
in
{ overlays ? overlaysDefault
, upstream ? nixpkgs
, ... } @ args:
import upstream ( builtins.removeAttrs args [ "upstream" ]
// { overlays = overlaysAlways ++ overlays; } )
self: super:
import ./pkgs/all-packages.nix self super

61
flake.lock generated Normal file
View File

@@ -0,0 +1,61 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1684854566,
"narHash": "sha256-Swfk4jyHzDoqZUGeYHTWC5jOJ2PI7yBiAZqLeG6+zVU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "daa7efafc2cf6e5efa7ff1c1d24f91127a2d8da4",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "release-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

32
flake.nix Normal file
View File

@@ -0,0 +1,32 @@
{
description = "Compute Canada Flake Packages (Overlay)";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
inputs.flake-utils.url = "github:numtide/flake-utils";
outputs = { self, nixpkgs, flake-utils }: {
overlays = {
default = final: prev: import ./pkgs/all-packages.nix final prev;
};
} // (flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
config.allowAliases = false;
overlays = [ self.overlays.default ];
};
inherit (pkgs) lib;
overlayAttributes = lib.pipe (import ./. pkgs pkgs) [
builtins.attrNames
(lib.partition (n: lib.isDerivation pkgs.${n}))
];
attributesToAttrset = attributes: lib.pipe attributes [
(map (n: lib.nameValuePair n pkgs.${n}))
lib.listToAttrs
];
in {
lib = attributesToAttrset overlayAttributes.wrong;
packages = attributesToAttrset overlayAttributes.right;
}
));
}

View File

@@ -1,8 +1,7 @@
# self - next package set (after our additions)
# super - prior package set (before our additions)
# pkgs - final package set (after everyone's additions)
# final - next package set (after our additions)
# prev - prior package set (before our additions)
#
self: super: pkgs: with pkgs; {
final: prev: with prev; {
fastqc = callPackage ./fastqc.nix { };
@@ -16,10 +15,11 @@ self: super: pkgs: with pkgs; {
#meraculous = callPackage ./meraculous.nix { let stdenv49 = overrideCC stdenv gcc49; in stdenv = stdenv49; boost = boost159.override { stdenv = stdenv49 }; };
mfixgui = python36Packages.callPackage ./mfix/mfixgui.nix { };
mfixgui = python38Packages.callPackage ./mfix/mfixgui.nix { };
mfixsolver = callPackage ./mfix/mfixsolver.nix { };
inherit (callPackage ./opengl/default.nix { inherit super; }) libGL mesa_glxgallium;
inherit (callPackage ./opengl/default.nix { inherit prev; }) libGL mesa_glxgallium;
openfoam = callPackage ./openfoam.nix { };
@@ -27,9 +27,9 @@ self: super: pkgs: with pkgs; {
ovito = libsForQt5.callPackage ./ovito.nix { };
pythonOverrides = lib.composeExtensions super.pythonOverrides (import ./python-packages.nix pkgs);
pythonOverrides = lib.composeExtensions prev.pythonOverrides (import ./python-packages.nix pkgs);
rOverrides = lib.composeExtensions super.rOverrides (import ./r-modules.nix pkgs);
rOverrides = lib.composeExtensions prev.rOverrides (import ./r-modules.nix pkgs);
smoove = callPackage ./smoove { };
@@ -41,12 +41,12 @@ self: super: pkgs: with pkgs; {
samblaster = callPackage ./samblaster.nix { };
svtyper = pythonPackages.callPackage ./svtyper.nix { };
svtyper = python38Packages.callPackage ./svtyper.nix { };
idba = callPackage ./idba.nix { };
nixStable = ( super.nixStable.override { confDir= "/nix/etc"; } ).overrideAttrs
( attrs: { patches = attrs.patches or [] ++ [ ./nix/remote.patch ./nix/cvmfs.patch ]; } );
nix = ( super.nix.override { confDir = "/nix/etc"; } ).overrideAttrs
# nixStable = ( prev.nixStable.override { confDir= "/nix/etc"; } ).overrideAttrs
# ( attrs: { patches = attrs.patches or [] ++ [ ./nix/remote.patch ./nix/cvmfs.patch ]; } );
nix = ( prev.nix.override { confDir = "/nix/etc"; } ).overrideAttrs
( attrs: { patches = attrs.patches or [] ++ [ ./nix/remote.patch ./nix/cvmfs.patch ]; } );
}

View File

@@ -1,4 +1,4 @@
{ stdenv, fetchsvn, autoreconfHook, pkgconfig
{ stdenv, fetchsvn, autoreconfHook, pkg-config
, libxml2, python2Packages, swig }:
stdenv.mkDerivation rec {
@@ -37,7 +37,7 @@ stdenv.mkDerivation rec {
buildInputs = [
autoreconfHook
pkgconfig
pkg-config
libxml2
python2Packages.python
swig

View File

@@ -1,6 +1,6 @@
{ stdenv, fetchFromGitHub, which, autoconf, automake
, zlib, curl
, python, hexdump, sambamba, samblaster, samtools }:
, python2, hexdump, sambamba, samblaster, samtools }:
stdenv.mkDerivation rec {
version = "0.3.0";
@@ -39,7 +39,7 @@ stdenv.mkDerivation rec {
automake
zlib
curl
(python.withPackages (packages: with packages; [ numpy pysam ]))
(python2.withPackages (packages: with packages; [ numpy pysam ]))
hexdump
sambamba
samblaster

View File

@@ -1,4 +1,4 @@
{ lib, fetchurl, fetchFromGitHub, buildPythonApplication, sphinx, sphinx_rtd_theme
{ lib, fetchurl, fetchFromGitHub, buildPythonApplication, sphinx, sphinx-rtd-theme
, mfixsolver
, numpy, psutil, pyqtgraph, qtpy, pyqt5, vtk, requests, simpleeval, simplejson }:
@@ -65,7 +65,7 @@ buildPythonApplication rec {
'';
nativeBuildInputs = [
sphinx sphinx_rtd_theme
sphinx sphinx-rtd-theme
];
buildInputs = [

View File

@@ -1,5 +1,5 @@
{ stdenv, fetchurl, fetchFromGitHub
, cmake, gfortran, python, pkgconfig
, cmake, gfortran, python38, pkg-config
, openmpi, netcdf, boost }:
stdenv.mkDerivation rec {
@@ -55,8 +55,8 @@ stdenv.mkDerivation rec {
];
nativeBuildInputs = [
python
pkgconfig
python38
pkg-config
];
buildInputs = [

View File

@@ -3,8 +3,8 @@
# Add a setup hook to the mesa package that automatically adds a
# libvglfaker.so dependency to executables that depend on libGL.so.
{ super, stdenv, buildEnv, substituteAll, bash
, autoreconfHook, pkgconfig, python2
{ prev, stdenv, buildEnv, substituteAll, bash
, autoreconfHook, pkg-config, python2
, xorg, llvmPackages, expat, mesa_glxgallium, mesa, libglvnd }:
let
@@ -13,23 +13,23 @@ let
# Ugliness required to break the loop created by the fact that
# the libGL hook requires VirtualGL which requires libGL.
#
# This would be clean if super was closed on itself (nixpkgs #15280)
# This would be clean if prev was closed on itself (nixpkgs #15280)
libGL = super.libGL;
libGLU = super.libGLU.override { inherit libGL; };
libGL = prev.libGL;
libGLU = prev.libGLU.override { inherit libGL; };
libGLU_combined = buildEnv {
name = "libGLU-combined";
paths = [ libGL libGLU ];
extraOutputsToInstall = [ "dev" ];
};
virtualglLib = (super.virtualglLib.override { inherit libGL libGLU fltk; }).overrideAttrs (attrs: {
virtualglLib = (prev.virtualglLib.override { inherit libGL libGLU fltk; }).overrideAttrs (attrs: {
postFixup = attrs.postFixup or "" + ''
patchelf --set-rpath /usr/lib${stdenv.lib.optionalString stdenv.is64bit "64"}/nvidia:"$(patchelf --print-rpath $out/lib/libvglfaker.so)" $out/lib/libvglfaker.so
'';
} );
fltk = super.fltk.override { inherit libGL libGLU freeglut; };
freeglut = super.freeglut.override { inherit libGL libGLU; };
fltk = prev.fltk.override { inherit libGL libGLU freeglut; };
freeglut = prev.freeglut.override { inherit libGL libGLU; };
in
substituteAll {
src = ./insert-virtualgl.sh;

View File

@@ -1,5 +1,5 @@
{ stdenv, autoreconfHook, pkgconfig, fetchFromGitHub
, zlib, openjpeg, libtiff, cairo, libpng, gdk_pixbuf, libxml2, sqlite }:
{ stdenv, autoreconfHook, pkg-config, fetchFromGitHub
, zlib, openjpeg, libtiff, cairo, libpng, gdk-pixbuf, libxml2, sqlite }:
stdenv.mkDerivation rec {
version = "3.4.1";
@@ -14,13 +14,13 @@ stdenv.mkDerivation rec {
buildInputs = [
autoreconfHook
pkgconfig
pkg-config
zlib
openjpeg
libtiff
cairo
libpng
gdk_pixbuf
gdk-pixbuf
libxml2
sqlite
];

View File

@@ -1,5 +1,5 @@
{ mkDerivation, lib, fetchFromGitLab, cmake
, boost, netcdf, hdf5, fftwSinglePrec, muparser, openssl, ffmpeg, python
, boost, netcdf, hdf5, fftwSinglePrec, muparser, openssl, ffmpeg, python38
, qtbase, qtsvg, qttools, qscintilla }:
mkDerivation rec {
@@ -21,7 +21,7 @@ mkDerivation rec {
muparser
openssl
ffmpeg
python
python38
qtbase qtsvg qttools qscintilla
];

View File

@@ -1,6 +1,6 @@
{ lib, fetchPypi, buildPythonApplication
, scipy, cytoolz, pysam
, pytest, pytestrunner }:
, pytest, pytest-runner }:
buildPythonApplication rec {
version = "0.7.1";
@@ -16,7 +16,7 @@ buildPythonApplication rec {
--replace o"pytest==4.6.4" "pytest"
'';
nativeBuildInputs = [ pytestrunner ];
nativeBuildInputs = [ pytest-runner ];
propagatedBuildInputs = [ scipy cytoolz pysam ];
checkInputs = [ pytest ];

View File

@@ -1,167 +0,0 @@
{ ccpkgs # Checked out ccpkgs git repo
? { outPath = ./.;
revCount = 1234 + 149;
shortRev = "gabcdef";
}
, nixpkgs # Checked out nixpkgs git repe
? { outPath = ./upstream;
revCount = 5678 + 242760;
shortRev = "gfedcba";
}
, nixpkgsArgs ? { config = { allowUnfree = true; inHydra = true; }; }
, stableBranch ? false
, supportedSystems ? [ "x86_64-linx" ] }:
# Free up the nixpkgs and ccpkgs name for binding to final product.
#
let
# *Git - git checkout
#
nixpkgsGit = nixpkgs;
ccpkgsGit = ccpkgs;
# Tarball hydra release product for the ccpkgs channel.
#
# 1. Generate an unpacked channel for the associated nixpkgs in the store.
# 2. Symlink this into the top-level directory as upstream (nixpkgs is used).
# 3. Stick in the standard version files.
# 4. Let releaseTools.sourceTarball handle the details of exposing it as a hydra product.
#
in let
# pkgs - Packages sucked in from the given ccpkgs git version.
#
pkgs = import ccpkgsGit ( nixpkgsArgs // { upstream = nixpkgsGit; } );
# version - Version number (e.g., "17.09")
# version*Suffix - Version suffix (e.g., "5678.gfedcba")
#
# Code lifted from nixos/release.nix.
#
version =
let
versionNixpkgs = pkgs.lib.fileContents (nixpkgsGit + /.version);
versionCcpkgs = pkgs.lib.fileContents ./.version;
in
assert (versionNixpkgs == versionCcpkgs); versionCcpkgs;
versionSeparator = if stableBranch then "." else "pre";
mkVersion = base: git: rec {
count = toString (git.revCount - base);
commit = git.shortRev;
suffix = "${versionSeparator}${count}.${commit}";
};
nixpkgsVersion = mkVersion 242760 nixpkgsGit;
ccpkgsVersion = mkVersion 149 ccpkgsGit;
versionSuffix = "${versionSeparator}${ccpkgsVersion.count}.${nixpkgsVersion.count}.${ccpkgsVersion.commit}.${nixpkgsVersion.commit}";
# nixpkgs - The store path containing the unpacked nixpkgs channel.
#
# 1. Generate a channel tarball from the git repo via the nixos/release.nix expression for hydra.
# 2. Unpack this into the store using the nix/unpack-channel.nix expression used by nix-channel.
#
nixpkgs =
let
# channel - Store path containing the channel release tarballs.
#
# The nixos.channel attribute from nixos/release.nix builds the nixpkgs channel tarballs.
#
channel =
( import (nixpkgsGit + /nixos/release.nix) {
inherit stableBranch supportedSystems;
nixpkgs = nixpkgsGit;
} ).channel;
# nixpkgs - The store path containing the unpacked nixpkgs channel.
#
# The nix provided nix/unpack-channel.nix function extracts the nixpkgs from the channel,
#
nixpkgs =
( pkgs.callPackage ./unpack-channel.nix { } rec {
name = "nixos";
channelName = "${name}-${version}";
src = channel + /tarballs + "/${name}-${version}${nixpkgsVersion.suffix}.tar.xz";
} );
in
nixpkgs;
# jobs - The jobs hydra is to run.
#
jobs =
let
# Extract the valid derivations from an overlay
#
# nameSet - the attribute set the uncomposed overlay (just the names are required)
# valueSet - the final attribute from the composed overlay (the values are required)
#
extractDerivations = valueSet: nameSet:
builtins.listToAttrs
( builtins.map
( name: { inherit name; value = valueSet.${name}; } )
( pkgs.lib.filter
( name: ( builtins.tryEval (pkgs.lib.isDerivation valueSet.${name}) ).value )
( pkgs.lib.attrNames nameSet ) ) );
# Provided overlays
pkgsNamesTopLevel = let self = import ./pkgs/all-packages.nix self { } self; in self;
pkgsNamesPython = let self = import ./pkgs/python-packages.nix { } self { } ; in self;
pkgsNamesR = let self = import ./pkgs/r-modules.nix { } self { } ; in self;
temporaryNamesTopLevel = let self = import ./temporary/all-packages.nix self { } self; in self;
temporaryNamesPython = let self = import ./temporary/python-packages.nix { } self { } ; in self;
temporaryNamesR = let self = import ./temporary/r-modules.nix { } self { } ; in self;
in {
# Tweak the nixos make-channel code to include the upstream channel.
#
# 1. Replace the git repo nixpkgs with a copy of the unpacked nixpkgs channel.
#
channel =
( import (nixpkgsGit + /nixos/lib/make-channel.nix) {
inherit pkgs version versionSuffix;
nixpkgs = ccpkgsGit;
} ).overrideAttrs ( original: {
name = "ccpkgs-channel";
distPhase =
''
rm -fr upstream
cp -rd ${nixpkgs}/nixos-${version} upstream
'' + original.distPhase;
} );
# tested - Aggregate package set required to built for automatic channel release
#
tested = pkgs.lib.hydraJob (pkgs.releaseTools.aggregate {
name = "ccpkgs-${version}";
constituents = builtins.map ( pkgs.lib.collect pkgs.lib.isDerivation ) [ jobs.pkgs jobs.temporary ];
meta = {
description = "Release-critical builds for the ccpkgs channel";
maintainers = with pkgs.lib.maintainers; [ ];
};
});
# pkgs - Attribute set of overlayed pkgs.
#
pkgs = extractDerivations pkgs pkgsNamesTopLevel // {
python2Packages = extractDerivations pkgs.python2Packages pkgsNamesPython;
python3Packages = extractDerivations pkgs.python3Packages pkgsNamesPython;
rPackages = extractDerivations pkgs.rPackages pkgsNamesR;
};
# temporary - Attribute set of overlayed pkgs.
#
temporary = extractDerivations pkgs temporaryNamesTopLevel // {
python2Packages = extractDerivations pkgs.python2Packages temporaryNamesPython;
python3Packages = extractDerivations pkgs.python3Packages temporaryNamesPython;
rPackages = extractDerivations pkgs.rPackages temporaryNamesR;
};
};
in jobs

View File

@@ -1,45 +0,0 @@
# Hydra runs the newer nix which doesn't have corepkgs/unpack-channel.nix. The nix-daemon runs the older nix which
# doesn't have builtins:unpack-channel. This means
#
# - for now we need to provide the older corepkgs/unpack-channel.nix
# - in the future we should switch to src/nix-channel/unpack-channel.nix
{ bash, coreutils, gnutar, xz, gzip, bzip2 }:
let
tarFlags = "--warning=no-timestamp";
builder = builtins.toFile "unpack-channel.sh"
''
mkdir $out
cd $out
xzpat="\.xz\$"
gzpat="\.gz\$"
if [[ "$src" =~ $xzpat ]]; then
xz -d < $src | tar xf - ${tarFlags}
elif [[ "$src" =~ $gzpat ]]; then
gzip -d < $src | tar xf - ${tarFlags}
else
bzip2 -d < $src | tar xf - ${tarFlags}
fi
if [ * != $channelName ]; then
mv * $out/$channelName
fi
'';
in
{ name, channelName, src }:
derivation {
system = builtins.currentSystem;
builder = "${bash}/bin/sh";
args = [ "-e" builder ];
inherit name channelName src;
PATH = "${coreutils}/bin:${gnutar}/bin:${xz}/bin:${gzip}/bin:${bzip2}/bin";
# No point in doing this remotely.
preferLocalBuild = true;
}