Add configuration of command completion scripts
to the Windows installer. This enables edkrepo command
completions to work "out of box" on Git for Windows by
adding the edkrepo command completions scripts to
the Git for Windows /etc/profile.d directory

Cc: Ashley DeSimone <ashley.e.desimone@...>
Cc: Puja Pandya <puja.pandya@...>
Cc: Erik Bjorge <erik.c.bjorge@...>
Cc: Prince Agyeman <prince.agyeman@...>
Cc: Bret Barkelew <Bret.Barkelew@...>
Cc: Philippe Mathieu-Daude <philmd@...>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@...>
---
.../EdkRepoInstaller/InstallWorker.cs | 212 ++++++++++++++++--
.../EdkRepoInstaller/InstallerStrings.cs | 44 +++-
.../Vendor/win_edkrepo_prompt.sh | 60 +++++
3 files changed, 296 insertions(+), 20 deletions(-)
create mode 100644 edkrepo_installer/Vendor/win_edkrepo_prompt.sh

diff --git a/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs b/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs
index 472a6c8..679b4f4 100644
--- a/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs
+++ b/edkrepo_installer/EdkRepoInstaller/InstallWorker.cs
@@ -19,6 +19,7 @@ using System.Security.AccessControl;
using System.Security.Cryptography;
using System.Security.Principal;
using System.Text;
+using System.Text.RegularExpressions;
using System.Threading;

namespace TianoCore.EdkRepoInstaller
@@ -611,7 +612,7 @@ namespace TianoCore.EdkRepoInstaller
SilentProcess process = SilentProcess.StartConsoleProcessSilently(GitPath, "--version", dataCapture.DataReceivedHandler);
process.WaitForExit();
PythonVersion gitVersion = new PythonVersion(dataCapture.GetData().Trim());
- if (gitVersion < new PythonVersion(2,13,0))
+ if (gitVersion < new PythonVersion(2, 13, 0))
{
InstallLogger.Log(string.Format("Git Version 2.13 or later is required. You have version {0}, please upgrade to a newer version of Git.", gitVersion));
ReportComplete(false, false);
@@ -624,7 +625,7 @@ namespace TianoCore.EdkRepoInstaller
List<Tuple<string, PythonVersion>> ExclusivePackages = new List<Tuple<string, PythonVersion>>();
foreach (PythonInstance PyInstance in PythonWheelsToInstall)
{
- foreach(PythonWheel Wheel in PyInstance.Wheels)
+ foreach (PythonWheel Wheel in PyInstance.Wheels)
{
if (Wheel.UninstallAllOtherCopies)
{
@@ -668,13 +669,13 @@ namespace TianoCore.EdkRepoInstaller
//
foreach (PythonVersion Obsolete in ObsoletedPythonVersions)
{
- if(ExistingEdkRepoPaths.Select(p => p.Item2).Contains(Obsolete))
+ if (ExistingEdkRepoPaths.Select(p => p.Item2).Contains(Obsolete))
{
foreach (string ExistingEdkrepoPythonPath in ExistingEdkRepoPaths.Where(p => p.Item2 == Obsolete).Select(p => p.Item1))
{
string UninstallerPath = Path.Combine(Path.GetDirectoryName(ExistingEdkrepoPythonPath), "Lib", "site-packages");
UninstallerPath = Path.Combine(UninstallerPath, Path.GetFileName(WindowsHelpers.GetApplicationPath()));
- if(File.Exists(UninstallerPath))
+ if (File.Exists(UninstallerPath))
{
InstallLogger.Log(string.Format("Uninstalling {0}...", UninstallerPath));
string UninstallString = string.Format("\"{0}\" /Uninstall /Passive", UninstallerPath);
@@ -788,14 +789,15 @@ namespace TianoCore.EdkRepoInstaller
//
// Step 10 - Setup symlink to edkrepo and bash script to launch edkrepo from git bash
//
+ string EdkrepoSymlinkPath = null;
if (!string.IsNullOrEmpty(EdkrepoPythonPath))
{
string EdkrepoScriptPath = Path.Combine(Path.GetDirectoryName(EdkrepoPythonPath), "Scripts", InstallerStrings.EdkrepoCliExecutable);
- string EdkrepoSymlinkPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), InstallerStrings.EdkrepoCliExecutable);
+ EdkrepoSymlinkPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), InstallerStrings.EdkrepoCliExecutable);
if (File.Exists(EdkrepoScriptPath))
{
bool CreateSymlink = true;
- if(File.Exists(EdkrepoSymlinkPath))
+ if (File.Exists(EdkrepoSymlinkPath))
{
try
{
@@ -805,22 +807,22 @@ namespace TianoCore.EdkRepoInstaller
}
}
catch (NotASymlinkException) { }
- if(CreateSymlink)
+ if (CreateSymlink)
{
File.Delete(EdkrepoSymlinkPath);
}
}
- if(CreateSymlink)
+ if (CreateSymlink)
{
InstallLogger.Log("Creating Symbolic Link for edkrepo.exe...");
WindowsHelpers.CreateSymbolicLink(EdkrepoSymlinkPath, EdkrepoScriptPath, WindowsHelpers.SYMBOLIC_LINK_FLAG.File);
}
string GitBashBinPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(GitPath)), "usr", "bin");
- if(Directory.Exists(GitBashBinPath))
+ if (Directory.Exists(GitBashBinPath))
{
InstallLogger.Log("Creating edkrepo launcher in Git Bash...");
string EdkrepoBashScriptPath = Path.Combine(GitBashBinPath, InstallerStrings.EdkrepoBashLauncherScript);
- if(File.Exists(EdkrepoBashScriptPath))
+ if (File.Exists(EdkrepoBashScriptPath))
{
File.Delete(EdkrepoBashScriptPath);
}
@@ -838,7 +840,7 @@ namespace TianoCore.EdkRepoInstaller
// Step 11 - Copy edkrepo config file to the edkrepo global data directory
//
string EdkrepoCfg = Path.Combine(Path.GetDirectoryName(WindowsHelpers.GetApplicationPath()), InstallerStrings.EdkrepoCfg);
- if(File.Exists(EdkrepoCfg))
+ if (File.Exists(EdkrepoCfg))
{
CreateEdkrepoGlobalDataDirectory();
string EdkrepoCfgDir = Path.Combine(WindowsHelpers.GetAllUsersAppDataPath(), InstallerStrings.EdkrepoGlobalDirectoryName);
@@ -853,7 +855,7 @@ namespace TianoCore.EdkRepoInstaller
{
string NewCfgHash = ComputeSha256(ReadFile(EdkrepoCfg));
string OldCfgHash = ComputeSha256(ReadFile(EdkrepoCfgTarget));
- if(NewCfgHash != OldCfgHash)
+ if (NewCfgHash != OldCfgHash)
{
if (GetPreviousEdkrepoCfgFileHashes().Contains(OldCfgHash))
{
@@ -908,7 +910,119 @@ namespace TianoCore.EdkRepoInstaller
}

//
- // Step 13 - Create Programs and Features uninstall links
+ // Step 13 - Copy win_edkrepo_prompt.sh and generate edkrepo_completions.sh
+ //
+ string edkrepoPromptSource = Path.Combine(Path.GetDirectoryName(WindowsHelpers.GetApplicationPath()), InstallerStrings.EdkrepoPrompt);
+
+ if (File.Exists(edkrepoPromptSource) && !string.IsNullOrEmpty(EdkrepoSymlinkPath))
+ {
+ string gitBashEtcPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(GitPath)), "etc");
+ string gitBashEtcProfileDPath = Path.Combine(gitBashEtcPath, "profile.d");
+ if (Directory.Exists(gitBashEtcPath) && Directory.Exists(gitBashEtcProfileDPath))
+ {
+ InstallLogger.Log("Installing EdkRepo command completion...");
+
+ //Copy win_edkrepo_prompt.sh
+ string edkrepoPromptDest = Path.Combine(gitBashEtcProfileDPath, InstallerStrings.EdkrepoPrompt);
+ if (File.Exists(edkrepoPromptDest))
+ {
+ File.Delete(edkrepoPromptDest);
+ }
+ File.Copy(edkrepoPromptSource, edkrepoPromptDest);
+ DirectoryInfo info = new DirectoryInfo(edkrepoPromptDest);
+ DirectorySecurity security = info.GetAccessControl();
+ security.AddAccessRule(new FileSystemAccessRule(
+ new SecurityIdentifier(WellKnownSidType.BuiltinUsersSid, null),
+ FileSystemRights.FullControl,
+ InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
+ PropagationFlags.NoPropagateInherit,
+ AccessControlType.Allow
+ ));
+ info.SetAccessControl(security);
+ InstallLogger.Log(string.Format("Copied {0}", InstallerStrings.EdkrepoPrompt));
+
+ //Generate edkrepo_completions.sh
+ string edkrepoCompletionDest = Path.Combine(gitBashEtcProfileDPath, InstallerStrings.EdkrepoCompletion);
+ if (File.Exists(edkrepoCompletionDest))
+ {
+ File.Delete(edkrepoCompletionDest);
+ }
+ dataCapture = new SilentProcess.StdoutDataCapture();
+ process = SilentProcess.StartConsoleProcessSilently(
+ EdkrepoSymlinkPath,
+ string.Format(
+ "generate-command-completion-script \"{0}\"",
+ edkrepoCompletionDest),
+ dataCapture.DataReceivedHandler);
+ process.WaitForExit();
+ InstallLogger.Log(EdkrepoSymlinkPath);
+ InstallLogger.Log(edkrepoCompletionDest);
+ InstallLogger.Log(dataCapture.GetData().Trim());
+ if (process.ExitCode != 0)
+ {
+ throw new InvalidOperationException(string.Format("generate-command-completion-script failed with status {0}", process.ExitCode));
+ }
+ if (!File.Exists(edkrepoCompletionDest))
+ {
+ throw new InvalidOperationException(string.Format("generate-command-completion-script did not create {0}", InstallerStrings.EdkrepoCompletion));
+ }
+ info = new DirectoryInfo(edkrepoCompletionDest);
+ security = info.GetAccessControl();
+ security.AddAccessRule(new FileSystemAccessRule(
+ new SecurityIdentifier(WellKnownSidType.BuiltinUsersSid, null),
+ FileSystemRights.FullControl,
+ InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
+ PropagationFlags.NoPropagateInherit,
+ AccessControlType.Allow
+ ));
+ info.SetAccessControl(security);
+ InstallLogger.Log(string.Format("Generated {0}", InstallerStrings.EdkrepoCompletion));
+
+ //Call win_edkrepo_prompt.sh from bash.bashrc so edkrepo completions are available for "interactive non-login" bash shells
+ string bashrcPath = Path.Combine(gitBashEtcPath, "bash.bashrc");
+ if (File.Exists(bashrcPath))
+ {
+ string bashrc = Encoding.UTF8.GetString(ReadFile(bashrcPath));
+ Match match = Regex.Match(bashrc, InstallerStrings.BashrcEdkrepoPromptCallPattern);
+ if (match.Success)
+ {
+ InstallLogger.Log("EdkRepo prompt is already in bash.bashrc");
+ }
+ else
+ {
+ bashrc = string.Format("{0}{1}", bashrc, InstallerStrings.BashrcEdkRepoPromptCall);
+ using (BinaryWriter writer = new BinaryWriter(File.Open(bashrcPath, FileMode.Truncate, FileAccess.Write)))
+ {
+ string sanitized = bashrc.Replace("\r\n", "\n");
+ writer.Write(Encoding.UTF8.GetBytes(sanitized));
+ }
+ InstallLogger.Log("EdkRepo prompt added to bash.bashrc");
+ }
+ }
+ else
+ {
+ InstallLogger.Log(string.Format("{0} not found", bashrcPath));
+ }
+ }
+ else
+ {
+ InstallLogger.Log("Git for Windows /etc/profile.d not found");
+ }
+ }
+ else
+ {
+ if (string.IsNullOrEmpty(EdkrepoSymlinkPath))
+ {
+ InstallLogger.Log("EdkRepo symlink not found");
+ }
+ if (!File.Exists(edkrepoPromptSource))
+ {
+ InstallLogger.Log(string.Format("{0} not found", InstallerStrings.EdkrepoPrompt));
+ }
+ }
+
+ //
+ // Step 14 - Create Programs and Features uninstall links
//
if (!string.IsNullOrEmpty(EdkrepoPythonPath))
{
@@ -977,7 +1091,7 @@ namespace TianoCore.EdkRepoInstaller
string GitPath = PythonOperations.GetFullPath("git.exe");

//
- // Step 2 - Delete symlink to edkrepo and bash script to launch it from git bash
+ // Step 2 - Delete symlink to edkrepo
//
string EdkrepoSymlinkPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), InstallerStrings.EdkrepoCliExecutable);
if (File.Exists(EdkrepoSymlinkPath))
@@ -1003,7 +1117,11 @@ namespace TianoCore.EdkRepoInstaller
File.Delete(EdkrepoSymlinkPath);
}
}
- if (GitPath != null)
+
+ //
+ // Step 3 - Delete scripts to launch edkrepo and Python from git bash, and edkrepo command completion scripts
+ //
+ if (!string.IsNullOrWhiteSpace(GitPath))
{
string GitBashBinPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(GitPath)), "usr", "bin");
if (Directory.Exists(GitBashBinPath))
@@ -1057,10 +1175,68 @@ namespace TianoCore.EdkRepoInstaller
File.Delete(EdkrepoPython2ScriptPath);
}
}
+ string gitBashEtcPath = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(GitPath)), "etc");
+ string gitBashEtcProfileDPath = Path.Combine(gitBashEtcPath, "profile.d");
+ if (Directory.Exists(gitBashEtcPath) && Directory.Exists(gitBashEtcProfileDPath))
+ {
+ string edkrepoPromptDest = Path.Combine(gitBashEtcProfileDPath, InstallerStrings.EdkrepoPrompt);
+ if (File.Exists(edkrepoPromptDest))
+ {
+ AllowCancel(false);
+ if (CancelPending())
+ {
+ ReportComplete(true, true);
+ return;
+ }
+ InstallLogger.Log(string.Format("Deleting {0}...", InstallerStrings.EdkrepoPrompt));
+ File.Delete(edkrepoPromptDest);
+ }
+
+ string edkrepoCompletionDest = Path.Combine(gitBashEtcProfileDPath, InstallerStrings.EdkrepoCompletion);
+ if (File.Exists(edkrepoCompletionDest))
+ {
+ AllowCancel(false);
+ if (CancelPending())
+ {
+ ReportComplete(true, true);
+ return;
+ }
+ InstallLogger.Log(string.Format("Deleting {0}...", InstallerStrings.EdkrepoCompletion));
+ File.Delete(edkrepoCompletionDest);
+ }
+
+ //Remove call win_edkrepo_prompt.sh from bash.bashrc
+ string bashrcPath = Path.Combine(gitBashEtcPath, "bash.bashrc");
+ if (File.Exists(bashrcPath))
+ {
+ string original_bashrc = Encoding.UTF8.GetString(ReadFile(bashrcPath));
+
+ string new_bashrc = Regex.Replace(original_bashrc, InstallerStrings.BashrcEdkrepoPromptCommentPattern, "");
+ new_bashrc = Regex.Replace(new_bashrc, InstallerStrings.BashrcEdkrepoPromptCallPattern, "");
+ if (new_bashrc == original_bashrc)
+ {
+ InstallLogger.Log("EdkRepo not found in bash.bashrc");
+ }
+ else
+ {
+ new_bashrc = new_bashrc.TrimEnd();
+ using (BinaryWriter writer = new BinaryWriter(File.Open(bashrcPath, FileMode.Truncate, FileAccess.Write)))
+ {
+ string sanitized = new_bashrc.Replace("\r\n", "\n");
+ writer.Write(Encoding.UTF8.GetBytes(sanitized));
+ }
+ InstallLogger.Log("EdkRepo prompt removed from bash.bashrc");
+ }
+ }
+ else
+ {
+ InstallLogger.Log(string.Format("{0} not found", bashrcPath));
+ }
+ }
}

//
- // Step 3 - Uninstall any instances of edkrepo
+ // Step 4 - Uninstall any instances of edkrepo
//
IEnumerable<string> PackagesToUninstall = GetPythonWheelsToUninstall();
InstallLogger.Log("Determining currently installed Python packages...");
@@ -1109,7 +1285,7 @@ namespace TianoCore.EdkRepoInstaller
}

//
- // Step 4 - Invoke the Finish Uninstall Event
+ // Step 5 - Invoke the Finish Uninstall Event
//
if (VendorCustomizer.Instance != null)
{
@@ -1122,7 +1298,7 @@ namespace TianoCore.EdkRepoInstaller
}

//
- // Step 5 - Delete Programs and Feature uninstall link
+ // Step 6 - Delete Programs and Feature uninstall link
//
RegistryKey hklm = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64);
RegistryKey winUninstallRegistryKey = hklm.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall", true);
diff --git a/edkrepo_installer/EdkRepoInstaller/InstallerStrings.cs b/edkrepo_installer/EdkRepoInstaller/InstallerStrings.cs
index 6aa90ee..1542641 100644
--- a/edkrepo_installer/EdkRepoInstaller/InstallerStrings.cs
+++ b/edkrepo_installer/EdkRepoInstaller/InstallerStrings.cs
@@ -2,7 +2,7 @@
InstallerStrings.cs

@copyright
- Copyright 2016 - 2019 Intel Corporation. All rights reserved.<BR>
+ Copyright 2016 - 2020 Intel Corporation. All rights reserved.<BR>
SPDX-License-Identifier: BSD-2-Clause-Patent

@par Specification Reference:
@@ -43,7 +43,7 @@ namespace TianoCore.EdkRepoInstaller
}

public static string EdkrepoPackageName
- {
+ {
get
{
return "edkrepo";
@@ -113,6 +113,46 @@ namespace TianoCore.EdkRepoInstaller
}
}

+ public static string EdkrepoPrompt
+ {
+ get
+ {
+ return "win_edkrepo_prompt.sh";
+ }
+ }
+
+ public static string EdkrepoCompletion
+ {
+ get
+ {
+ return "edkrepo_completions.sh";
+ }
+ }
+
+ public static string BashrcEdkrepoPromptCommentPattern
+ {
+ get
+ {
+ return @"#\s+Install\s+EdkRepo\s+command\s+completions";
+ }
+ }
+
+ public static string BashrcEdkrepoPromptCallPattern
+ {
+ get
+ {
+ return @"shopt\s+-q\s+login_shell\s+\|\|\s+\.\s+/etc/profile\.d/win_edkrepo_prompt\.sh";
+ }
+ }
+
+ public static string BashrcEdkRepoPromptCall
+ {
+ get
+ {
+ return "\n\n# Install EdkRepo command completions\nshopt -q login_shell || . /etc/profile.d/win_edkrepo_prompt.sh";
+ }
+ }
+
public static string InstallerName
{
get
diff --git a/edkrepo_installer/Vendor/win_edkrepo_prompt.sh b/edkrepo_installer/Vendor/win_edkrepo_prompt.sh
new file mode 100644
index 0000000..5404175
--- /dev/null
+++ b/edkrepo_installer/Vendor/win_edkrepo_prompt.sh
@@ -0,0 +1,60 @@
+## @file win_edkrepo_prompt.sh
+# Note: For use on Git for Windows/MSYS2 ONLY.
+# UNIX version is in install.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+# Add EdkRepo command completions
+[[ -r "/etc/profile.d/edkrepo_completions.sh" ]] && . "/etc/profile.d/edkrepo_completions.sh"
+
+# Add EdkRepo to the prompt
+ps1len="${#PS1}"
+let "pos38 = ps1len - 38"
+let "pos3 = ps1len - 3"
+let "pos2 = ps1len - 2"
+if [ "${PS1:pos38}" == '\[\033[36m\]`__git_ps1`\[\033[0m\]\n$ ' ]; then
+ newps1="${PS1:0:pos38}"
+ prompt_suffix='\[\033[36m\]`__git_ps1`\[\033[0m\]\n$ '
+elif [ "${PS1:pos3}" == "\\$ " ]; then
+ newps1="${PS1:0:pos3}"
+ prompt_suffix="\\$ "
+elif [ "${PS1:pos3}" == " $ " ]; then
+ newps1="${PS1:0:pos3}"
+ prompt_suffix=" $ "
+elif [ "${PS1:pos2}" == "$ " ]; then
+ newps1="${PS1:0:pos2}"
+ prompt_suffix="$ "
+else
+ newps1="$PS1"
+ prompt_suffix=""
+fi
+
+if [ -x "$(command -v edkrepo)" ] && [ -x "$(command -v $command_completion_edkrepo_file)" ]; then
+ newps1="$newps1\[\033[32m\]\$current_edkrepo_combo\[\033[00m\]"
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+
+ # Determining the current Edkrepo combo requires invoking Python and parsing
+ # manifest XML, which is a relatively expensive operation to do every time
+ # the user presses <Enter>.
+ # As a performance optimization, only do this if the present working directory
+ # changed
+ if [[ ! -z ${PROMPT_COMMAND+x} ]] && [[ "$PROMPT_COMMAND" != "edkrepo_combo_chpwd" ]]; then
+ old_prompt_command=$PROMPT_COMMAND
+ fi
+ old_pwd=$(pwd)
+ edkrepo_combo_chpwd() {
+ if [[ "$(pwd)" != "$old_pwd" ]]; then
+ old_pwd=$(pwd)
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+ fi
+ if [[ ! -z ${PROMPT_COMMAND+x} ]]; then
+ eval $old_prompt_command
+ fi
+ }
+ PROMPT_COMMAND=edkrepo_combo_chpwd
+fi
+
+PS1="$newps1$prompt_suffix"
+MSYS2_PS1="$PS1" # for detection by MSYS2 SDK's bash.basrc
--
2.24.0.windows.2

Add configuration of command completion scripts
to install.py This enables edkrepo command completions
to work "out of box" on most Linux systems by appending
to the user's .bashrc and .zshrc startup scripts
inclusion of the EdkRepo command completion scripts.

Cc: Ashley DeSimone <ashley.e.desimone@...>
Cc: Puja Pandya <puja.pandya@...>
Cc: Erik Bjorge <erik.c.bjorge@...>
Cc: Prince Agyeman <prince.agyeman@...>
Cc: Bret Barkelew <Bret.Barkelew@...>
Cc: Philippe Mathieu-Daude <philmd@...>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@...>
---
edkrepo_installer/linux-scripts/install.py | 285 ++++++++++++++++++++-
1 file changed, 283 insertions(+), 2 deletions(-)

diff --git a/edkrepo_installer/linux-scripts/install.py b/edkrepo_installer/linux-scripts/install.py
index b2cdfed..52f0c52 100755
--- a/edkrepo_installer/linux-scripts/install.py
+++ b/edkrepo_installer/linux-scripts/install.py
@@ -15,6 +15,7 @@ import importlib.util
import logging
import os
import platform
+import re
import stat
import shutil
import subprocess
@@ -22,7 +23,7 @@ import sys
import traceback
import xml.etree.ElementTree as et

-tool_sign_on = 'Installer for edkrepo version {}\nCopyright(c) Intel Corporation, 2019'
+tool_sign_on = 'Installer for edkrepo version {}\nCopyright(c) Intel Corporation, 2020'

# Data here should be maintained in a configuration file
cfg_dir = '.edkrepo'
@@ -31,6 +32,21 @@ cfg_src_dir = os.path.abspath('config')
whl_src_dir = os.path.abspath('wheels')
def_python = 'python3'

+# ZSH Configuration options
+prompt_regex = re.compile(r"#\s+[Aa][Dd][Dd]\s+[Ee][Dd][Kk][Rr][Ee][Pp][Oo]\s+&\s+[Gg][Ii][Tt]\s+[Tt][Oo]\s+[Tt][Hh][Ee]\s+[Pp][Rr][Oo][Mm][Pp][Tt]")
+zsh_autoload_compinit_regex = re.compile(r"autoload\s+-U\s+compinit")
+zsh_autoload_bashcompinit_regex = re.compile(r"autoload\s+-U\s+bashcompinit")
+zsh_autoload_colors_regex = re.compile(r"autoload\s+-U\s+colors")
+zsh_colors_regex = re.compile(r"\n\s*colors\n")
+zsh_compinit_regex = re.compile(r"compinit\s+-u")
+zsh_bashcompinit_regex = re.compile(r"\n\s*bashcompinit\n")
+zsh_autoload_compinit = 'autoload -U compinit'
+zsh_autoload_bashcompinit = 'autoload -U bashcompinit'
+zsh_autoload_colors = 'autoload -U colors'
+zsh_colors = 'colors'
+zsh_compinit = 'compinit -u'
+zsh_bashcompinit = 'bashcompinit'
+
def default_run(cmd):
return subprocess.run(cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)

@@ -51,8 +67,85 @@ def get_args():
parser.add_argument('-p', '--py', action='store', default=None, help='Specify the python command to use when installing')
parser.add_argument('-u', '--user', action='store', default=None, help='Specify user account to install edkrepo support on')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Enables verbose output')
+ parser.add_argument('--no-prompt', action='store_true', default=False, help='Do NOT add EdkRepo combo and git branch to the shell prompt')
+ parser.add_argument('--prompt', action='store_true', default=False, help='Add EdkRepo combo and git branch to the shell prompt')
return parser.parse_args()

+def is_prompt_customization_installed(user_home_dir):
+ script_files = [os.path.join(user_home_dir, '.bashrc'), os.path.join(user_home_dir, '.zshrc')]
+ customization_installed = True
+ for script_file in script_files:
+ if os.path.isfile(script_file):
+ with open(script_file, 'r') as f:
+ script = f.read().strip()
+ data = prompt_regex.search(script)
+ if not data:
+ customization_installed = False
+ break
+ return customization_installed
+
+__add_prompt_customization = None
+def get_add_prompt_customization(args, user_home_dir):
+ global __add_prompt_customization
+ if __add_prompt_customization is not None:
+ return __add_prompt_customization
+ if args.no_prompt:
+ __add_prompt_customization = False
+ return False
+ elif args.prompt:
+ __add_prompt_customization = True
+ return True
+ #Check if the prompt customization has already been installed
+ if is_prompt_customization_installed(user_home_dir):
+ __add_prompt_customization = False
+ return False
+ #If the prompt has not been installed and EdkRepo >= 2.0.0 is installed, then don't install the prompt customization
+ if shutil.which('edkrepo') is not None:
+ res = default_run(['edkrepo', '--version'])
+ if _check_version(res.stdout.replace('edkrepo ', '').strip(), '2.0.0') >= 0:
+ __add_prompt_customization = False
+ return False
+ #Show the user an advertisement to see if they want the prompt customization
+ from select import select
+ import termios
+ import tty
+ def get_key(timeout=-1):
+ key = None
+ old_settings = termios.tcgetattr(sys.stdin.fileno())
+ try:
+ tty.setraw(sys.stdin.fileno())
+ if timeout != -1:
+ rlist, _, _ = select([sys.stdin], [], [], timeout)
+ if rlist:
+ key = sys.stdin.read(1)
+ else:
+ key = sys.stdin.read(1)
+ finally:
+ termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, old_settings)
+ return key
+ print('\nEdkRepo can show the checked out \033[32mcombo\033[00m and \033[36mbranch\033[00m as part of the command prompt')
+ print('For example, instead of:\n')
+ print('\033[01;32muser@machine\033[00m:\033[01;34mEdk2\033[00m$ ')
+ print('\nThe command prompt would look like:\n')
+ print('\033[01;32muser@machine\033[00m:\033[01;34mEdk2\033[00m \033[32m[Edk2Master]\033[36m (master)\033[00m$ ')
+ print('')
+ while True:
+ print('Would you like the combo and branch shown on the command prompt? [y/N] ')
+ key = get_key(120)
+ if key:
+ if key == 'y' or key == 'Y':
+ print('Y')
+ __add_prompt_customization = True
+ return True
+ if key == 'n' or key == 'N':
+ print('N')
+ __add_prompt_customization = False
+ return False
+ else:
+ print('No response after 2min... assuming no.')
+ __add_prompt_customization = False
+ return False
+
def get_installed_packages(python_command):
pip_cmd = [def_python, '-m', 'pip', 'list', '--legacy']
try:
@@ -146,6 +239,176 @@ def set_execute_permissions():
stat_data = os.stat(py_file)
os.chmod(py_file, stat_data.st_mode | stat.S_IEXEC)

+bash_prompt_customization = r'''
+# Add EdkRepo & git to the prompt
+ps1len="${#PS1}"
+let "pos3 = ps1len - 3"
+let "pos2 = ps1len - 2"
+if [ "${PS1:pos3}" == "\\$ " ]; then
+ newps1="${PS1:0:pos3}"
+ prompt_suffix="\\$ "
+elif [ "${PS1:pos3}" == " $ " ]; then
+ newps1="${PS1:0:pos3}"
+ prompt_suffix=" $ "
+elif [ "${PS1:pos2}" == "$ " ]; then
+ newps1="${PS1:0:pos2}"
+ prompt_suffix="$ "
+else
+ newps1="$PS1"
+ prompt_suffix=""
+fi
+
+# EdkRepo combo in prompt.
+if [ -x "$(command -v edkrepo)" ] && [ -x "$(command -v command_completion_edkrepo)" ]; then
+ newps1="$newps1\[\033[32m\]\$current_edkrepo_combo"
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+
+ # Determining the current Edkrepo combo requires invoking Python and parsing
+ # manifest XML, which is a relatively expensive operation to do every time
+ # the user presses <Enter>.
+ # As a performance optimization, only do this if the present working directory
+ # changed
+ if [[ ! -z ${PROMPT_COMMAND+x} ]] && [[ "$PROMPT_COMMAND" != "edkrepo_combo_chpwd" ]]; then
+ old_prompt_command=$PROMPT_COMMAND
+ fi
+ old_pwd=$(pwd)
+ edkrepo_combo_chpwd() {
+ if [[ "$(pwd)" != "$old_pwd" ]]; then
+ old_pwd=$(pwd)
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+ fi
+ if [[ ! -z ${PROMPT_COMMAND+x} ]]; then
+ eval $old_prompt_command
+ fi
+ }
+ PROMPT_COMMAND=edkrepo_combo_chpwd
+fi
+
+# Git branch in prompt.
+parse_git_branch() {
+ git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/'
+}
+
+PS1="$newps1\[\033[36m\]\$(parse_git_branch)\[\033[00m\]$prompt_suffix"
+'''
+
+zsh_prompt_customization = r'''
+# Add EdkRepo & git to the prompt
+prompt_length="${#PROMPT}"
+let "pos4 = prompt_length - 3"
+let "pos3 = prompt_length - 2"
+if [ "${PROMPT[$pos4,$prompt_length]}" = ' %# ' ]; then
+ new_prompt="${PROMPT[1,$pos4-1]}"
+ prompt_suffix=" %# "
+elif [ "${PROMPT[$pos3,$prompt_length]}" = "%# " ]; then
+ new_prompt="${PROMPT[1,$pos3-1]}"
+ prompt_suffix="%# "
+else
+ new_prompt="$PROMPT"
+ prompt_suffix=""
+fi
+
+# EdkRepo combo in prompt.
+if [ -x "$(command -v edkrepo)" ] && [ -x "$(command -v command_completion_edkrepo)" ]; then
+ new_prompt="$new_prompt%{$fg[green]%}\$current_edkrepo_combo%{$reset_color%}"
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+
+ # Determining the current Edkrepo combo requires invoking Python and parsing
+ # manifest XML, which is a relatively expensive operation to do every time
+ # the user presses <Enter>.
+ # As a performance optimization, only do this if the present working directory
+ # changed
+ function edkrepo_combo_chpwd() {
+ current_edkrepo_combo=$(command_completion_edkrepo current-combo)
+ }
+ chpwd_functions=(${chpwd_functions[@]} "edkrepo_combo_chpwd")
+fi
+
+# Load version control information
+autoload -Uz vcs_info
+precmd() { vcs_info }
+
+# Format the vcs_info_msg_0_ variable
+zstyle ':vcs_info:git:*' formats " %{$fg[cyan]%}(%b)%{$reset_color%}"
+
+# Set up the prompt (with git branch name)
+setopt PROMPT_SUBST
+eval "PROMPT='$new_prompt\${vcs_info_msg_0_}\$prompt_suffix'"
+'''
+
+def add_command_to_startup_script(script_file, regex, command, username):
+ script = ''
+ if os.path.isfile(script_file):
+ with open(script_file, 'r') as f:
+ script = f.read().strip()
+ data = regex.search(script)
+ if not data:
+ if script == '':
+ script = command
+ else:
+ script = '{}\n{}'.format(script, command)
+ with open(script_file, 'w') as f:
+ f.write(script)
+
+def add_command_comment_to_startup_script(script_file, regex, command, comment, username):
+ script = ''
+ if os.path.isfile(script_file):
+ with open(script_file, 'r') as f:
+ script = f.read().strip()
+ (new_script, subs) = re.subn(regex, command, script)
+ if subs == 0:
+ command = '\n{1}\n{0}\n'.format(command, comment)
+ if script == '':
+ new_script = command
+ else:
+ new_script = '{}\n{}'.format(script, command)
+ if new_script != script:
+ with open(script_file, 'w') as f:
+ f.write(new_script)
+ shutil.chown(script_file, user=username)
+ os.chmod(script_file, 0o644)
+
+def add_command_completions_to_shell(command_completion_script, args, username, user_home_dir):
+ # Add "source ~/.bashrc" to ~/.bash_profile if it does not have it already
+ bash_profile_file = os.path.join(user_home_dir, '.bash_profile')
+ bash_profile = ''
+ if os.path.isfile(bash_profile_file):
+ with open(bash_profile_file, 'r') as f:
+ bash_profile = f.read().strip()
+ profile_source_regex = re.compile(r"source\s+~/\.bashrc")
+ profile_source_regex2 = re.compile(r".\s+~/\.bashrc")
+ data = profile_source_regex.search(bash_profile)
+ if not data:
+ data = profile_source_regex2.search(bash_profile)
+ if not data:
+ if bash_profile == '':
+ bash_profile = 'source ~/.bashrc\n'
+ else:
+ bash_profile = '{}\nsource ~/.bashrc\n'.format(bash_profile)
+ with open(bash_profile_file, 'w') as f:
+ f.write(bash_profile)
+ shutil.chown(bash_profile_file, user=username)
+ os.chmod(bash_profile_file, 0o644)
+
+ # Add edkrepo command completion to ~/.bashrc if it does not have it already
+ regex = r"\[\[\s+-r\s+\"\S*edkrepo_completions.sh\"\s+\]\]\s+&&\s+.\s+\"\S*edkrepo_completions.sh\""
+ new_source_line = '[[ -r "{0}" ]] && . "{0}"'.format(command_completion_script)
+ comment = '\n# Add EdkRepo command completions'
+ bash_rc_file = os.path.join(user_home_dir, '.bashrc')
+ add_command_comment_to_startup_script(bash_rc_file, regex, new_source_line, comment, username)
+ if get_add_prompt_customization(args, user_home_dir):
+ add_command_to_startup_script(bash_rc_file, prompt_regex, bash_prompt_customization, username)
+ zsh_rc_file = os.path.join(user_home_dir, '.zshrc')
+ add_command_to_startup_script(zsh_rc_file, zsh_autoload_compinit_regex, zsh_autoload_compinit, username)
+ add_command_to_startup_script(zsh_rc_file, zsh_autoload_bashcompinit_regex, zsh_autoload_bashcompinit, username)
+ add_command_to_startup_script(zsh_rc_file, zsh_autoload_colors_regex, zsh_autoload_colors, username)
+ add_command_to_startup_script(zsh_rc_file, zsh_colors_regex, zsh_colors, username)
+ add_command_to_startup_script(zsh_rc_file, zsh_compinit_regex, zsh_compinit, username)
+ add_command_to_startup_script(zsh_rc_file, zsh_bashcompinit_regex, zsh_bashcompinit, username)
+ add_command_comment_to_startup_script(zsh_rc_file, regex, new_source_line, comment, username)
+ if get_add_prompt_customization(args, user_home_dir):
+ add_command_to_startup_script(zsh_rc_file, prompt_regex, zsh_prompt_customization, username)
+
def do_install():
global def_python
org_python = None
@@ -209,6 +472,7 @@ def do_install():
log.info('- Unable to determine users home directory')
return 1
default_cfg_dir = os.path.join(user_home_dir, cfg_dir)
+ get_add_prompt_customization(args, user_home_dir)
log.info('+ System information collected')

# Display current system information.
@@ -348,7 +612,7 @@ def do_install():
#Delete obsolete dependencies
if updating_edkrepo:
installed_packages = get_installed_packages(def_python)
- for whl_name in ['smmap2', 'gitdb2']:
+ for whl_name in ['smmap2', 'gitdb2', 'edkrepo-internal']:
if whl_name in installed_packages:
try:
res = default_run([def_python, '-m', 'pip', 'uninstall', '--yes', whl_name])
@@ -375,6 +639,23 @@ def do_install():
set_execute_permissions()
log.info('+ Marked scripts as executable')

+
+ #Install the command completion script
+ if shutil.which('edkrepo') is not None:
+ if args.local:
+ command_completion_script = os.path.join(default_cfg_dir, 'edkrepo_completions.sh')
+ else:
+ command_completion_script = os.path.join('/', 'etc', 'profile.d', 'edkrepo_completions.sh')
+ try:
+ res = default_run(['edkrepo', 'generate-command-completion-script', command_completion_script])
+ if args.local:
+ shutil.chown(command_completion_script, user=username)
+ os.chmod(command_completion_script, 0o644)
+ add_command_completions_to_shell(command_completion_script, args, username, user_home_dir)
+ except:
+ log.info('- Failed to configure edkrepo command completion')
+ if args.verbose:
+ traceback.print_exc()
log.log(logging.PRINT, '\nInstallation complete\n')

return 0
--
2.24.0.windows.2

Adds code to edkrepo_cli.py to generate a bash/zsh
compatible command completion script.

Add a new command_completion_edkrepo.py script which
is callable by the shell when needed for dynamic
completion data. For example, providing completion for
"edkrepo checkout" requires the shell to know the list
of possible branch combinations, this requires parsing
the manifest XML. command_completion_edkrepo.py provides
a means for the shell to get that type of data.

Cc: Ashley DeSimone <ashley.e.desimone@...>
Cc: Puja Pandya <puja.pandya@...>
Cc: Erik Bjorge <erik.c.bjorge@...>
Cc: Prince Agyeman <prince.agyeman@...>
Cc: Bret Barkelew <Bret.Barkelew@...>
Cc: Philippe Mathieu-Daude <philmd@...>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@...>
---
edkrepo/command_completion_edkrepo.py | 86 +++++++++++++++++++++++++++
edkrepo/edkrepo_cli.py | 61 ++++++++++++++++++-
setup.py | 8 +--
3 files changed, 150 insertions(+), 5 deletions(-)
create mode 100644 edkrepo/command_completion_edkrepo.py

diff --git a/edkrepo/command_completion_edkrepo.py b/edkrepo/command_completion_edkrepo.py
new file mode 100644
index 0000000..05de9ca
--- /dev/null
+++ b/edkrepo/command_completion_edkrepo.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+## @file
+# command_completion_edkrepo.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import argparse
+import io
+import os
+import sys
+import traceback
+
+from edkrepo_manifest_parser.edk_manifest import ManifestXml
+from edkrepo.config import config_factory
+from edkrepo.config.config_factory import get_workspace_manifest
+
+def checkout(parsed_args, config):
+ manifest = get_workspace_manifest()
+ print(' '.join([c.name for c in manifest.combinations]))
+
+def current_combo(parsed_args, config):
+ manifest = get_workspace_manifest()
+ print(" [{}]".format(manifest.general_config.current_combo))
+
+def checkout_pin(parsed_args, config):
+ pins = []
+ manifest_directory = config['cfg_file'].manifest_repo_abs_local_path
+ manifest = get_workspace_manifest()
+ pin_folder = os.path.normpath(os.path.join(manifest_directory, manifest.general_config.pin_path))
+ for dirpath, _, filenames in os.walk(pin_folder):
+ for file in filenames:
+ pin_file = os.path.join(dirpath, file)
+ # Capture error output from manifest parser stdout so it is hidden unless verbose is enabled
+ stdout = sys.stdout
+ sys.stdout = io.StringIO()
+ pin = ManifestXml(pin_file)
+ parse_output = sys.stdout.getvalue()
+ sys.stdout = stdout
+ if parsed_args.verbose and parse_output.strip() != '':
+ print('Pin {} Parsing Errors: {}\n'.format(file, parse_output.strip()))
+ if pin.project_info.codename == manifest.project_info.codename:
+ pins.append(file)
+ print(' '.join(pins))
+
+# To add command completions for a new command, add an entry to this dictionary.
+command_completions = {
+ 'current-combo': current_combo,
+ 'checkout': checkout,
+ 'checkout-pin': checkout_pin,
+ 'chp': checkout_pin
+}
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-v", "--verbose", action="store_true", help='Increases command verbosity')
+ subparsers = parser.add_subparsers(dest='subparser_name')
+ for command_completion in command_completions:
+ subparsers.add_parser(command_completion, formatter_class=argparse.RawTextHelpFormatter)
+ if len(sys.argv) <= 1:
+ return 0
+ parsed_args = parser.parse_args()
+ try:
+ command_name = parsed_args.subparser_name
+ config = {}
+ config["cfg_file"] = config_factory.GlobalConfig()
+ config["user_cfg_file"] = config_factory.GlobalUserConfig()
+ if command_name not in command_completions:
+ return 1
+ command_completions[command_name](parsed_args, config)
+ return 0
+ except Exception as e:
+ if parsed_args.verbose:
+ traceback.print_exc()
+ print("Error: {}".format(str(e)))
+ return 1
+ return 0
+
+if __name__ == "__main__":
+ try:
+ sys.exit(main())
+ except Exception as e:
+ traceback.print_exc()
+ sys.exit(1)
diff --git a/edkrepo/edkrepo_cli.py b/edkrepo/edkrepo_cli.py
index 0b69860..03061c9 100644
--- a/edkrepo/edkrepo_cli.py
+++ b/edkrepo/edkrepo_cli.py
@@ -3,7 +3,7 @@
## @file
# edkrepo_cli.py
#
-# Copyright (c) 2017- 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#

@@ -29,6 +29,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoException, EdkrepoGlobalConf
from edkrepo.common.edkrepo_exception import EdkrepoWarningException
from edkrepo.common.edkrepo_exception import EdkrepoConfigFileInvalidException
from edkrepo.common.humble import KEYBOARD_INTERRUPT, GIT_CMD_ERROR
+from edkrepo.common.pathfix import get_actual_path

def generate_command_line(command):
parser = argparse.ArgumentParser()
@@ -100,6 +101,61 @@ def generate_command_line(command):
subparser_name.add_argument(('--' + arg.get('name')), action=arg_action, help=arg.get('help-text'))
return parser

+command_completion_script_header='''#!/usr/bin/env bash
+#
+## @file edkrepo_completions.sh
+#
+# Automatically generated please DO NOT modify !!!
+#
+
+'''
+def generate_command_completion_script(script_filename, parser):
+ import edkrepo.command_completion_edkrepo as completion
+ commands = []
+ for action in parser._positionals._group_actions:
+ if action.choices is not None:
+ commands = [c for c in action.choices]
+ break
+ commands = sorted(commands)
+ commands_with_3rd_param_completion = [c for c in completion.command_completions if c in commands]
+ commands_with_3rd_param_completion = sorted(commands_with_3rd_param_completion)
+ with open(script_filename, 'w') as f:
+ f.write(command_completion_script_header)
+ if sys.platform == "win32":
+ command_completion_path = os.path.dirname(sys.executable)
+ command_completion_path = os.path.join(command_completion_path, 'Scripts', "command_completion_edkrepo.exe")
+ if not os.path.isfile(command_completion_path):
+ print('command_completion_edkrepo.exe not found')
+ return
+ command_completion_path = get_actual_path(command_completion_path)
+ (drive, path) = os.path.splitdrive(command_completion_path)
+ command_completion_path = '/{}{}'.format(drive.replace(':','').lower(), path.replace('\\','/'))
+ f.write("export command_completion_edkrepo_file='{}'\n".format(command_completion_path))
+ f.write('alias command_completion_edkrepo="$command_completion_edkrepo_file"\n')
+ f.write('_edkrepo_completions() {\n if [ "${#COMP_WORDS[@]}" -eq "2" ]; then\n')
+ f.write(' COMPREPLY=($(compgen -W "{}" -- "${{COMP_WORDS[1]}}"))\n'.format(' '.join(commands)))
+ if len(commands_with_3rd_param_completion) > 0:
+ f.write(' elif [ "${#COMP_WORDS[@]}" -eq "3" ]; then\n')
+ first_loop = True
+ for command in commands_with_3rd_param_completion:
+ if first_loop:
+ f.write(' if [ "${{COMP_WORDS[1]}}" == "{}" ]; then\n'.format(command))
+ first_loop = False
+ else:
+ f.write(' elif [ "${{COMP_WORDS[1]}}" == "{}" ]; then\n'.format(command))
+ f.write(' COMPREPLY=($(compgen -W "$(command_completion_edkrepo ${COMP_WORDS[1]})" -- "${COMP_WORDS[2]}"))\n')
+ if len(commands_with_3rd_param_completion) > 0:
+ f.write(' fi\n')
+ f.write(' fi\n}\n\n')
+ if len(commands_with_3rd_param_completion) > 0:
+ if sys.platform == "win32":
+ f.write('if [ -x "$(command -v edkrepo)" ] && [ -x "$(command -v $command_completion_edkrepo_file)" ]; then\n')
+ else:
+ f.write('if [ -x "$(command -v edkrepo)" ] && [ -x "$(command -v command_completion_edkrepo)" ]; then\n')
+ else:
+ f.write('if [ -x "$(command -v edkrepo)" ]; then\n')
+ f.write(' complete -F _edkrepo_completions edkrepo\nfi\n')
+
def main():
command = command_factory.create_composite_command()
config = {}
@@ -117,6 +173,9 @@ def main():
if len(sys.argv) <= 1:
parser.print_help()
return 1
+ if sys.argv[1] == 'generate-command-completion-script' and len(sys.argv) >= 3:
+ generate_command_completion_script(sys.argv[2], parser)
+ return 0
parsed_args = parser.parse_args()
command_name = parsed_args.subparser_name
try:
diff --git a/setup.py b/setup.py
index e14aed1..e7e6ce8 100755
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,8 @@
#!/usr/bin/env python3
#
-## @file
-# setup.py
+## @file setup.py
#
-# Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2017 - 2020, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#

@@ -20,7 +19,8 @@ setup(name='edkrepo',
include_package_data=True,
entry_points={
'console_scripts': [
- 'edkrepo = edkrepo.edkrepo_entry_point:main'
+ 'edkrepo = edkrepo.edkrepo_entry_point:main',
+ 'command_completion_edkrepo = edkrepo.command_completion_edkrepo:main'
]
}
)
--
2.24.0.windows.2

This patch series adds command completions (also referred to as
[TAB] completions) to EdkRepo. It also adds the currently
checked out branch combination to the command prompt. This is a
significant quality of life improvement for EdkRepo's users.

The bash and zsh shells are supported. The reason why bash and
zsh were choosen is based on their popularity in the UNIX world.
zsh is the default shell on macOS and most contemporary Linux
distributions use bash as the default shell. Git for Windows
also uses bash.

Performance was a strong consideration in the design of this
feature. Since EdkRepo has become a fairly large amount of
Python code with lots of dependencies, the "boot time" for
the Python interperter to load and execute the EdkRepo
entrypoint is approximately 1-2 seconds. A 2 second delay
on TAB completion would not be a good user experience.
To mitigate this, instead of enumerating and instantiating
all the command classes to generate the list of EdkRepo
sub-commands every time the user presses the TAB
key, this is done once and stored in a auto-generated shell
script at installation time. This way, the shell has all the
information needed for static completions without having
to invoke the Python interperter at all.

There are cases where TAB completions need some dynamic data.
For example, completing the 3rd parameter after
"edkrepo checkout " requires the shell to know the list of
branch combinations, this requires parsing the manifest XML.
command_completion_edkrepo.py provides a means for the shell
to get that type of data. A new command_completion_edkrepo.py
script has been added for this purpose.
command_completion_edkrepo.py is callable by the shell when
dynamic completion data is needed.
command_completion_edkrepo.py has been tuned to limit imports
as much as possible so that it executes quickly.

command_completion_edkrepo.py is also used to retrieve the
currently checked out branch combination so that it can be
added to the prompt. For performance considerations,
this operation is only done iff the present working directory
was changed by the last command the shell executed.

Cc: Ashley DeSimone <ashley.e.desimone@...>
Cc: Puja Pandya <puja.pandya@...>
Cc: Erik Bjorge <erik.c.bjorge@...>
Cc: Prince Agyeman <prince.agyeman@...>
Cc: Bret Barkelew <Bret.Barkelew@...>
Cc: Philippe Mathieu-Daude <philmd@...>
Signed-off-by: Nate DeSimone <nathaniel.l.desimone@...>

Nate DeSimone (3):
EdkRepo: Generate command completion scripts
EdkRepo: Add command completion setup to install.py
EdkRepo: Add command completion setup to Windows installer

edkrepo/command_completion_edkrepo.py | 86 ++++++
edkrepo/edkrepo_cli.py | 61 +++-
.../EdkRepoInstaller/InstallWorker.cs | 212 +++++++++++--
.../EdkRepoInstaller/InstallerStrings.cs | 44 ++-
.../Vendor/win_edkrepo_prompt.sh | 60 ++++
edkrepo_installer/linux-scripts/install.py | 285 +++++++++++++++++-
setup.py | 8 +-
7 files changed, 729 insertions(+), 27 deletions(-)
create mode 100644 edkrepo/command_completion_edkrepo.py
create mode 100644 edkrepo_installer/Vendor/win_edkrepo_prompt.sh

--
2.24.0.windows.2

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Reviewed-by: Ashley DeSimone <ashley.e.desimone@...>

Hi Pankaj,

I have now finished my review of individual patches.

Beyond that, the build fails at the patch
10/28 "Silicon/NXP: remove print information from Soc lib" with the error
edk2-platforms/Silicon/NXP/Library/SocLib/Chassis.c:209:1: error:
‘CpuMaskNext’ defined but not used [-Werror=unused-function]

The function is deleted in the subsequent patch,
11/28 "Silicon/NXP: remove not needed components".
Please move that deletion to the now failing patch.

Regards,

Leif

On 3/30/20 7:47 PM, Dong, Eric wrote:

Hi Tom,
Sorry for late response. It’s a huge patch, please give me two more weeks to detail review them.
I have rough go through these patches and have some basic comments for them now:
1.It’s better to spit patch if changes files not in same package. Like patch 1/42.

Ok, will do.

2.All functions need to have comments for them. Miss comments in patch 10/42 and others.

Just external functions or both external and internal (STATIC) functions, too?

Thanks,
Tom

Please update patches to fix above basic checks first.
Thanks,
Eric
*From:*devel@edk2.groups.io [mailto:devel@edk2.groups.io] *On Behalf Of *Lendacky, Thomas
*Sent:* Tuesday, March 31, 2020 12:54 AM
*To:* devel@edk2.groups.io
*Cc:* Justen, Jordan L <jordan.l.justen@...>; Laszlo Ersek <lersek@...>; Ard Biesheuvel <ard.biesheuvel@...>; Kinney, Michael D <michael.d.kinney@...>; Gao, Liming <liming.gao@...>; Dong, Eric <eric.dong@...>; Ni, Ray <ray.ni@...>; Brijesh Singh <brijesh.singh@...>; You, Benjamin <benjamin.you@...>; Bi, Dandan <dandan.bi@...>; Dong, Guo <guo.dong@...>; Wu, Hao A <hao.a.wu@...>; Wang, Jian J <jian.j.wang@...>; Ma, Maurice <maurice.ma@...>
*Subject:* Re: [edk2-devel] [PATCH v6 00/42] SEV-ES guest support
I've gotten some nice feedback from Laszlo, especially on the OvmfPkg side
of this patchset, but haven't seen much response from the other
maintainers. Is there any feedback on the MdePkg, MdeModulePkg and
UefiCpuPkg changes that needs to be addressed in order to merge this?
I do have some minor changes on ensuring the per-CPU variable page stays
encrypted, but not much beyond that. Those changes can be submitted
afterwards or as a new version before inclusion.
Thanks,
Tom
On 3/24/20 12:40 PM, Tom Lendacky wrote:

 This patch series provides support for running EDK2/OVMF under SEV-ES.
 Secure Encrypted Virtualization - Encrypted State (SEV-ES) expands on the
 SEV support to protect the guest register state from the hypervisor. See
 "AMD64 Architecture Programmer's Manual Volume 2: System Programming",
 section "15.35 Encrypted State (SEV-ES)" [1].
 In order to allow a hypervisor to perform functions on behalf of a guest,
 there is architectural support for notifying a guest's operating system
 when certain types of VMEXITs are about to occur. This allows the guest to
 selectively share information with the hypervisor to satisfy the requested
 function. The notification is performed using a new exception, the VMM
 Communication exception (#VC). The information is shared through the
 Guest-Hypervisor Communication Block (GHCB) using the VMGEXIT instruction.
 The GHCB format and the protocol for using it is documented in "SEV-ES
 Guest-Hypervisor Communication Block Standardization" [2].
 The main areas of the EDK2 code that are updated to support SEV-ES are
 around the exception handling support and the AP boot support.
 Exception support is required starting in Sec, continuing through Pei
 and into Dxe in order to handle #VC exceptions that are generated.  Each
 AP requires it's own GHCB page as well as a page to hold values specific
 to that AP.
 AP booting poses some interesting challenges. The INIT-SIPI-SIPI sequence
 is typically used to boot the APs. However, the hypervisor is not allowed
 to update the guest registers. The GHCB document [2] talks about how SMP
 booting under SEV-ES is performed.
 Since the GHCB page must be a shared (unencrypted) page, the processor
 must be running in long mode in order for the guest and hypervisor to
 communicate with each other. As a result, SEV-ES is only supported under
 the X64 architecture.
 [1] https://www.amd.com/system/files/TechDocs/24593.pdf

<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.amd.com%2Fsystem%2Ffiles%2FTechDocs%2F24593.pdf&data=02%7C01%7Cthomas.lendacky%40amd.com%7C2ee33c1d932a4906558f08d7d50d1ca2%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637212125835211690&sdata=Q%2BIjeq%2FRDgiovKtPeA4TGDVorCK07jQVNZ7N9kvD%2BuE%3D&reserved=0>

 [2] https://developer.amd.com/wp-content/resources/56421.pdf

<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.amd.com%2Fwp-content%2Fresources%2F56421.pdf&data=02%7C01%7Cthomas.lendacky%40amd.com%7C2ee33c1d932a4906558f08d7d50d1ca2%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637212125835221679&sdata=bos02T0YR3i5xji9rhjPl7jpS5uJPKt1Q0hhdy%2FoMR0%3D&reserved=0>

 ---
 These patches are based on commit:
 2f524a745e23 ("BaseTools:Fix build tools print traceback info issue")
 Proper execution of SEV-ES relies on Bugzilla 2340 being fixed.
 A version of the tree (with an extra patch to workaround Bugzilla 2340) can
 be found at:
https://github.com/AMDESE/ovmf/tree/sev-es-v13

<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2FAMDESE%2Fovmf%2Ftree%2Fsev-es-v13&data=02%7C01%7Cthomas.lendacky%40amd.com%7C2ee33c1d932a4906558f08d7d50d1ca2%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637212125835221679&sdata=fmIyS5QBB7YGDSqTFiBIe%2BBdH1zatcEplUdNC2wi%2Fhc%3D&reserved=0>

 Cc: Ard Biesheuvel <ard.biesheuvel@... <mailto:ard.biesheuvel@...>>
 Cc: Benjamin You <benjamin.you@... <mailto:benjamin.you@...>>
 Cc: Dandan Bi <dandan.bi@... <mailto:dandan.bi@...>>
 Cc: Eric Dong <eric.dong@... <mailto:eric.dong@...>>
 Cc: Guo Dong <guo.dong@... <mailto:guo.dong@...>>
 Cc: Hao A Wu <hao.a.wu@... <mailto:hao.a.wu@...>>
 Cc: Jian J Wang <jian.j.wang@... <mailto:jian.j.wang@...>>
 Cc: Jordan Justen <jordan.l.justen@... <mailto:jordan.l.justen@...>>
 Cc: Laszlo Ersek <lersek@... <mailto:lersek@...>>
 Cc: Liming Gao <liming.gao@... <mailto:liming.gao@...>>
 Cc: Maurice Ma <maurice.ma@... <mailto:maurice.ma@...>>
 Cc: Michael D Kinney <michael.d.kinney@... <mailto:michael.d.kinney@...>>
 Cc: Ray Ni <ray.ni@... <mailto:ray.ni@...>>
 Changes since v5:
 - Remove extraneous VmgExitLib usage
 - Miscellaneous changes to address feedback (coding style, etc.)
 Changes since v4:
 - Move the SEV-ES protocol negotiation out of the SEC exception handler
    and into the SecMain.c file. As a result:
    - Move the SecGhcb related PCDs out of UefiCpuPkg and into OvmfPkg
    - Combine SecAMDSevVcHandler.c and PeiDxeAMDSevVcHandler.c into a
      single AMDSevVcHandler.c
 - Consolidate VmgExitLib usage into common LibraryClasses sections
 - Add documentation comments to the VmgExitLib functions
 Changes since v3:
 - Remove the need for the MP library finalization routine. The AP
    jump table address will be held by the hypervisor rather than
    communicated via the GHCB MSR. This removes some fragility around
    the UEFI to OS transition.
 - Rename the SEV-ES RIP reset area to SEV-ES workarea and use it to
    communicate the SEV-ES status, so that SEC CPU exception handling is
    only established for an SEV-ES guest.
 - Fix SMM build breakageAdd around QemuFlashPtrWrite().
 - Fix SMM build breakage by adding VC exception support the SMM CPU
    exception handling.
 - Add memory fencing around the invocation of AsmVmgExit().
 - Clarify comments around the SEV-ES AP reset RIP values and usage.
 - Move some PCD definitions from MdeModulePkg to UefiCpuPkg.
 - Remove the 16-bit code selector definition from MdeModulePkg
 Changes since v2:
 - Added a way to locate the SEV-ES fixed AP RIP address for starting
    AP's to avoid updating the actual flash image (build time location
    that is identified with a GUID value).
 - Create a VmgExit library to replace static inline functions.
 - Move some PCDs to the appropriate packages
 - Add support for writing to QEMU flash under SEV-ES
 - Add additional MMIO opcode support
 - Cleaned up the GHCB MSR CPUID protocol support
 Changes since v1:
 - Patches reworked to be more specific to the component/area being updated
    and order of definition/usage
 - Created a library for VMGEXIT-related functions to replace use of inline
    functions
 - Allocation method for GDT changed from AllocatePool to AllocatePages
 - Early caching only enabled for SEV-ES guests
 - Ensure AP loop mode set to halt loop mode for SEV-ES guests
 - Reserved SEC GHCB-related memory areas when S3 is enabled
 Tom Lendacky (42):
    MdePkg: Create PCDs to be used in support of SEV-ES
    MdePkg: Add the MSR definition for the GHCB register
    MdePkg: Add a structure definition for the GHCB
    MdeModulePkg/DxeIplPeim: Support GHCB pages when creating page tables
    MdePkg/BaseLib: Add support for the XGETBV instruction
    MdePkg/BaseLib: Add support for the VMGEXIT instruction
    UefiCpuPkg: Implement library support for VMGEXIT
    OvmfPkg: Prepare OvmfPkg to use the VmgExitLib library
    UefiPayloadPkg: Prepare UefiPayloadPkg to use the VmgExitLib library
    UefiCpuPkg/CpuExceptionHandler: Add base support for the #VC exception
    UefiCpuPkg/CpuExceptionHandler: Add support for IOIO_PROT NAE events
    UefiCpuPkg/CpuExceptionHandler: Support string IO for IOIO_PROT NAE
      events
    UefiCpuPkg/CpuExceptionHandler: Add support for CPUID NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for MSR_PROT NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for NPF NAE events (MMIO)
    UefiCpuPkg/CpuExceptionHandler: Add support for WBINVD NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for RDTSC NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for RDPMC NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for INVD NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for VMMCALL NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for RDTSCP NAE events
    UefiCpuPkg/CpuExceptionHandler: Add support for MONITOR/MONITORX NAE
      events
    UefiCpuPkg/CpuExceptionHandler: Add support for MWAIT/MWAITX NAE
      events
    UefiCpuPkg/CpuExceptionHandler: Add support for DR7 Read/Write NAE
      events
    OvmfPkg/MemEncryptSevLib: Add an SEV-ES guest indicator function
    OvmfPkg: Add support to perform SEV-ES initialization
    OvmfPkg: Create a GHCB page for use during Sec phase
    OvmfPkg/PlatformPei: Reserve GHCB-related areas if S3 is supported
    OvmfPkg: Create GHCB pages for use during Pei and Dxe phase
    OvmfPkg/PlatformPei: Move early GDT into ram when SEV-ES is enabled
    UefiCpuPkg: Create an SEV-ES workarea PCD
    OvmfPkg: Reserve a page in memory for the SEV-ES usage
    OvmfPkg/ResetVector: Add support for a 32-bit SEV check
    OvmfPkg/Sec: Add #VC exception handling for Sec phase
    OvmfPkg/Sec: Enable cache early to speed up booting
    OvmfPkg/QemuFlashFvbServicesRuntimeDxe: Bypass flash detection with
      SEV-ES is enabled
    UefiCpuPkg: Add a 16-bit protected mode code segment descriptor
    UefiCpuPkg/MpInitLib: Add CPU MP data flag to indicate if SEV-ES is
      enabled
    UefiCpuPkg: Allow AP booting under SEV-ES
    OvmfPkg: Use the SEV-ES work area for the SEV-ES AP reset vector
    OvmfPkg: Move the GHCB allocations into reserved memory
    UefiCpuPkg/MpInitLib: Prepare SEV-ES guest APs for OS use
   MdeModulePkg/MdeModulePkg.dec                 |    9 +
   OvmfPkg/OvmfPkg.dec                           |    9 +
   UefiCpuPkg/UefiCpuPkg.dec                     |   17 +
   OvmfPkg/OvmfPkgIa32.dsc                       |    6 +
   OvmfPkg/OvmfPkgIa32X64.dsc                    |    6 +
   OvmfPkg/OvmfPkgX64.dsc                        |    6 +
   OvmfPkg/OvmfXen.dsc                           |    1 +
   UefiCpuPkg/UefiCpuPkg.dsc                     |    2 +
   UefiPayloadPkg/UefiPayloadPkgIa32.dsc         |    2 +
   UefiPayloadPkg/UefiPayloadPkgIa32X64.dsc      |    2 +
   OvmfPkg/OvmfPkgX64.fdf                        |    9 +
   MdeModulePkg/Core/DxeIplPeim/DxeIpl.inf       |    2 +
   MdePkg/Library/BaseLib/BaseLib.inf            |    4 +
   OvmfPkg/PlatformPei/PlatformPei.inf           |    7 +
   .../FvbServicesRuntimeDxe.inf                 |    2 +
   OvmfPkg/ResetVector/ResetVector.inf           |    8 +
   OvmfPkg/Sec/SecMain.inf                       |    4 +
   .../DxeCpuExceptionHandlerLib.inf             |    5 +
   .../PeiCpuExceptionHandlerLib.inf             |    5 +
   .../SecPeiCpuExceptionHandlerLib.inf          |    5 +
   .../SmmCpuExceptionHandlerLib.inf             |    5 +
   UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf |    4 +
   UefiCpuPkg/Library/MpInitLib/PeiMpInitLib.inf |    4 +
   UefiCpuPkg/Library/VmgExitLib/VmgExitLib.inf  |   33 +
   .../Core/DxeIplPeim/X64/VirtualMemory.h       |   12 +-
   MdePkg/Include/Library/BaseLib.h              |   31 +
   MdePkg/Include/Register/Amd/Fam17Msr.h        |   42 +
   MdePkg/Include/Register/Amd/Ghcb.h            |  136 ++
   OvmfPkg/Include/Library/MemEncryptSevLib.h    |   12 +
   .../QemuFlash.h                               |    6 +
   UefiCpuPkg/CpuDxe/CpuGdt.h                    |    4 +-
   UefiCpuPkg/Include/Library/VmgExitLib.h       |  111 ++
   .../CpuExceptionHandlerLib/AMDSevVcCommon.h   |   26 +
   .../CpuExceptionCommon.h                      |    2 +
   UefiCpuPkg/Library/MpInitLib/MpLib.h          |   68 +-
   .../Core/DxeIplPeim/Ia32/DxeLoadFunc.c        |    4 +-
   .../Core/DxeIplPeim/X64/DxeLoadFunc.c         |   11 +-
   .../Core/DxeIplPeim/X64/VirtualMemory.c       |   49 +-
   MdePkg/Library/BaseLib/Ia32/GccInline.c       |   45 +
   MdePkg/Library/BaseLib/X64/GccInline.c        |   47 +
   .../MemEncryptSevLibInternal.c                |   75 +-
   OvmfPkg/PlatformPei/AmdSev.c                  |   82 ++
   OvmfPkg/PlatformPei/MemDetect.c               |   23 +
   .../QemuFlash.c                               |   23 +-
   .../QemuFlashDxe.c                            |   15 +
   .../QemuFlashSmm.c                            |    9 +
   OvmfPkg/Sec/SecMain.c                         |  160 ++-
   UefiCpuPkg/CpuDxe/CpuGdt.c                    |    8 +-
   .../CpuExceptionHandlerLib/AMDSevVcHandler.c  |   29 +
   .../CpuExceptionCommon.c                      |    2 +-
   .../Ia32/ArchAMDSevVcHandler.c                |   24 +
   .../PeiDxeSmmCpuException.c                   |   16 +
   .../SecPeiCpuException.c                      |   16 +
   .../X64/ArchAMDSevVcHandler.c                 | 1237 +++++++++++++++++
   UefiCpuPkg/Library/MpInitLib/DxeMpLib.c       |  114 +-
   UefiCpuPkg/Library/MpInitLib/MpLib.c          |  257 +++-
   UefiCpuPkg/Library/MpInitLib/PeiMpLib.c       |   19 +
   UefiCpuPkg/Library/VmgExitLib/VmgExitLib.c    |  249 ++++
   UefiCpuPkg/PiSmmCpuDxeSmm/X64/SmmFuncsArch.c  |    2 +-
   MdePkg/Library/BaseLib/Ia32/VmgExit.nasm      |   37 +
   MdePkg/Library/BaseLib/Ia32/XGetBv.nasm       |   31 +
   MdePkg/Library/BaseLib/X64/VmgExit.nasm       |   32 +
   MdePkg/Library/BaseLib/X64/XGetBv.nasm        |   34 +
   OvmfPkg/ResetVector/Ia16/ResetVectorVtf0.asm  |  100 ++
   OvmfPkg/ResetVector/Ia32/PageTables64.asm     |  351 ++++-
   OvmfPkg/ResetVector/ResetVector.nasmb         |   20 +
   .../X64/ExceptionHandlerAsm.nasm              |   17 +
   UefiCpuPkg/Library/MpInitLib/Ia32/MpEqu.inc   |    2 +-
   .../Library/MpInitLib/Ia32/MpFuncs.nasm       |   15 +
   UefiCpuPkg/Library/MpInitLib/X64/MpEqu.inc    |    4 +-
   UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm |  370 ++++-
   UefiCpuPkg/Library/VmgExitLib/VmgExitLib.uni  |   15 +
   .../ResetVector/Vtf0/Ia16/Real16ToFlat32.asm  |    9 +
   73 files changed, 4061 insertions(+), 99 deletions(-)
   create mode 100644 UefiCpuPkg/Library/VmgExitLib/VmgExitLib.inf
   create mode 100644 MdePkg/Include/Register/Amd/Ghcb.h
   create mode 100644 UefiCpuPkg/Include/Library/VmgExitLib.h
   create mode 100644 UefiCpuPkg/Library/CpuExceptionHandlerLib/AMDSevVcCommon.h
   create mode 100644 UefiCpuPkg/Library/CpuExceptionHandlerLib/AMDSevVcHandler.c
   create mode 100644 UefiCpuPkg/Library/CpuExceptionHandlerLib/Ia32/ArchAMDSevVcHandler.c
   create mode 100644 UefiCpuPkg/Library/CpuExceptionHandlerLib/X64/ArchAMDSevVcHandler.c
   create mode 100644 UefiCpuPkg/Library/VmgExitLib/VmgExitLib.c
   create mode 100644 MdePkg/Library/BaseLib/Ia32/VmgExit.nasm
   create mode 100644 MdePkg/Library/BaseLib/Ia32/XGetBv.nasm
   create mode 100644 MdePkg/Library/BaseLib/X64/VmgExit.nasm
   create mode 100644 MdePkg/Library/BaseLib/X64/XGetBv.nasm
   create mode 100644 OvmfPkg/ResetVector/Ia16/ResetVectorVtf0.asm
   create mode 100644 UefiCpuPkg/Library/VmgExitLib/VmgExitLib.uni

On Fri, Mar 20, 2020 at 20:05:42 +0530, Pankaj Bansal wrote:

From: Pankaj Bansal <pankaj.bansal@...>

MemoryInitPeiLib would be linked to MemoryInitPeim, when we implement
PEI phase. therefore, move the library to directory of same name.

Signed-off-by: Pankaj Bansal <pankaj.bansal@...>

Reviewed-by: Leif Lindholm <leif@...>

---
.../{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.c | 0
.../{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.h | 0
.../{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.inf | 0
Silicon/NXP/NxpQoriqLs.dsc.inc | 3 +--
4 files changed, 1 insertion(+), 2 deletions(-)
rename Silicon/NXP/Library/{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.c (100%)
rename Silicon/NXP/Library/{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.h (100%)
rename Silicon/NXP/Library/{MemoryInitPei => MemoryInitPeiLib}/MemoryInitPeiLib.inf (100%)

diff --git a/Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.c b/Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.c
similarity index 100%
rename from Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.c
rename to Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.c
diff --git a/Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.h b/Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.h
similarity index 100%
rename from Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.h
rename to Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.h
diff --git a/Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.inf b/Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.inf
similarity index 100%
rename from Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.inf
rename to Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.inf
diff --git a/Silicon/NXP/NxpQoriqLs.dsc.inc b/Silicon/NXP/NxpQoriqLs.dsc.inc
index 5f77f47f0399..b2b10ce28a93 100644
--- a/Silicon/NXP/NxpQoriqLs.dsc.inc
+++ b/Silicon/NXP/NxpQoriqLs.dsc.inc
@@ -102,6 +102,7 @@ [LibraryClasses.common]
PciLib|MdePkg/Library/BasePciLibPciExpress/BasePciLibPciExpress.inf

PlatformPeiLib|Silicon/NXP/Library/PlatformPeiLib/PlatformPeiLib.inf
+ MemoryInitPeiLib|Silicon/NXP/Library/MemoryInitPeiLib/MemoryInitPeiLib.inf

[LibraryClasses.common.SEC]
PcdLib|MdePkg/Library/BasePcdLibNull/BasePcdLibNull.inf
@@ -113,7 +114,6 @@ [LibraryClasses.common.SEC]
PrePiHobListPointerLib|ArmPlatformPkg/Library/PrePiHobListPointerLib/PrePiHobListPointerLib.inf
MemoryAllocationLib|EmbeddedPkg/Library/PrePiMemoryAllocationLib/PrePiMemoryAllocationLib.inf
PerformanceLib|MdeModulePkg/Library/PeiPerformanceLib/PeiPerformanceLib.inf
- MemoryInitPeiLib|Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.inf

# 1/123 faster than Stm or Vstm version
BaseMemoryLib|MdePkg/Library/BaseMemoryLib/BaseMemoryLib.inf
@@ -143,7 +143,6 @@ [LibraryClasses.common.DXE_DRIVER]
DxeServicesLib|MdePkg/Library/DxeServicesLib/DxeServicesLib.inf
SecurityManagementLib|MdeModulePkg/Library/DxeSecurityManagementLib/DxeSecurityManagementLib.inf
PerformanceLib|MdeModulePkg/Library/DxePerformanceLib/DxePerformanceLib.inf
- MemoryInitPeiLib|Silicon/NXP/Library/MemoryInitPei/MemoryInitPeiLib.inf

[LibraryClasses.common.UEFI_APPLICATION]
PcdLib|MdePkg/Library/DxePcdLib/DxePcdLib.inf
--
2.17.1

On Fri, Mar 20, 2020 at 20:05:41 +0530, Pankaj Bansal wrote:

From: Pankaj Bansal <pankaj.bansal@...>

Add VarStore Fd. This Fd is used to store non volatile variables in
flash.

Signed-off-by: Pankaj Bansal <pankaj.bansal@...>

Reviewed-by: Leif Lindholm <leif@...>

---
Platform/NXP/LS1043aRdbPkg/LS1043aRdbPkg.fdf | 3 +-
Platform/NXP/LS1043aRdbPkg/VarStore.fdf.inc | 91 ++++++++++++++++++++
2 files changed, 93 insertions(+), 1 deletion(-)
create mode 100644 Platform/NXP/LS1043aRdbPkg/VarStore.fdf.inc

diff --git a/Platform/NXP/LS1043aRdbPkg/LS1043aRdbPkg.fdf b/Platform/NXP/LS1043aRdbPkg/LS1043aRdbPkg.fdf
index 8d66f36d7407..99fbc87e1200 100644
--- a/Platform/NXP/LS1043aRdbPkg/LS1043aRdbPkg.fdf
+++ b/Platform/NXP/LS1043aRdbPkg/LS1043aRdbPkg.fdf
@@ -3,7 +3,7 @@
# FLASH layout file for LS1043a board.
#
# Copyright (c) 2016, Freescale Ltd. All rights reserved.
-# Copyright 2017-2019 NXP
+# Copyright 2017-2020 NXP
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
@@ -49,6 +49,7 @@ [FD.LS1043ARDB_EFI]
FV = FVMAIN_COMPACT

!include Platform/NXP/FVRules.fdf.inc
+!include VarStore.fdf.inc
################################################################################
#
# FV Section
diff --git a/Platform/NXP/LS1043aRdbPkg/VarStore.fdf.inc b/Platform/NXP/LS1043aRdbPkg/VarStore.fdf.inc
new file mode 100644
index 000000000000..391e4ae5eaf8
--- /dev/null
+++ b/Platform/NXP/LS1043aRdbPkg/VarStore.fdf.inc
@@ -0,0 +1,91 @@
+## @file
+# FDF include file with FD definition that defines an empty variable store.
+#
+# Copyright (c) 2006 - 2013, Intel Corporation. All rights reserved.
+# Copyright (C) 2014, Red Hat, Inc.
+# Copyright (c) 2016, Linaro, Ltd. All rights reserved.
+# Copyright (c) 2016, Freescale Semiconductor. All rights reserved.
+# Copyright 2017-2020 NXP
+#
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+##
+
+[FD.LS1043aRdbNv_EFI]
+BaseAddress = 0x60500000 #The base address of the FLASH device
+Size = 0x000C0000 #The size in bytes of the FLASH device
+ErasePolarity = 1
+BlockSize = 0x40000
+NumBlocks = 0x3
+
+#
+# Place NV Storage just above Platform Data Base
+#
+DEFINE NVRAM_AREA_VARIABLE_BASE = 0x00000000
+DEFINE NVRAM_AREA_VARIABLE_SIZE = 0x00040000
+DEFINE FTW_WORKING_BASE = $(NVRAM_AREA_VARIABLE_BASE) + $(NVRAM_AREA_VARIABLE_SIZE)
+DEFINE FTW_WORKING_SIZE = 0x00040000
+DEFINE FTW_SPARE_BASE = $(FTW_WORKING_BASE) + $(FTW_WORKING_SIZE)
+DEFINE FTW_SPARE_SIZE = 0x00040000
+
+#############################################################################
+# LS1043ARDB NVRAM Area
+# LS1043ARDB NVRAM Area contains: Variable + FTW Working + FTW Spare
+#############################################################################
+
+
+$(NVRAM_AREA_VARIABLE_BASE)|$(NVRAM_AREA_VARIABLE_SIZE)
+gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageVariableBase64|gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageVariableSize
+#NV_VARIABLE_STORE
+DATA = {
+ ## This is the EFI_FIRMWARE_VOLUME_HEADER
+ # ZeroVector []
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ # FileSystemGuid: gEfiSystemNvDataFvGuid =
+ # { 0xFFF12B8D, 0x7696, 0x4C8B,
+ # { 0xA9, 0x85, 0x27, 0x47, 0x07, 0x5B, 0x4F, 0x50 }}
+ 0x8D, 0x2B, 0xF1, 0xFF, 0x96, 0x76, 0x8B, 0x4C,
+ 0xA9, 0x85, 0x27, 0x47, 0x07, 0x5B, 0x4F, 0x50,
+ # FvLength: 0xC0000
+ 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00,
+ # Signature "_FVH" # Attributes
+ 0x5f, 0x46, 0x56, 0x48, 0x36, 0x0E, 0x00, 0x00,
+ # HeaderLength # CheckSum # ExtHeaderOffset #Reserved #Revision
+ 0x48, 0x00, 0xC2, 0xF9, 0x00, 0x00, 0x00, 0x02,
+ # Blockmap[0]: 0x3 Blocks * 0x40000 Bytes / Block
+ 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
+ # Blockmap[1]: End
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ ## This is the VARIABLE_STORE_HEADER
+ # It is compatible with SECURE_BOOT_ENABLE == FALSE as well.
+ # Signature: gEfiVariableGuid =
+ # { 0xddcf3616, 0x3275, 0x4164,
+ # { 0x98, 0xb6, 0xfe, 0x85, 0x70, 0x7f, 0xfe, 0x7d }}
+ 0x16, 0x36, 0xcf, 0xdd, 0x75, 0x32, 0x64, 0x41,
+ 0x98, 0xb6, 0xfe, 0x85, 0x70, 0x7f, 0xfe, 0x7d,
+ # Size: 0x40000 (gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageVariableSize) -
+ # 0x48 (size of EFI_FIRMWARE_VOLUME_HEADER) = 0x3ffb8
+ # This can speed up the Variable Dispatch a bit.
+ 0xB8, 0xFF, 0x03, 0x00,
+ # FORMATTED: 0x5A #HEALTHY: 0xFE #Reserved: UINT16 #Reserved1: UINT32
+ 0x5A, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+}
+
+$(FTW_WORKING_BASE)|$(FTW_WORKING_SIZE)
+gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageFtwWorkingBase64|gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageFtwWorkingSize
+#NV_FTW_WORKING
+DATA = {
+ # EFI_FAULT_TOLERANT_WORKING_BLOCK_HEADER->Signature = gEdkiiWorkingBlockSignatureGuid =
+ # { 0x9e58292b, 0x7c68, 0x497d, { 0xa0, 0xce, 0x65, 0x0, 0xfd, 0x9f, 0x1b, 0x95 }}
+ 0x2b, 0x29, 0x58, 0x9e, 0x68, 0x7c, 0x7d, 0x49,
+ 0xa0, 0xce, 0x65, 0x0, 0xfd, 0x9f, 0x1b, 0x95,
+ # Crc:UINT32 #WorkingBlockValid:1, WorkingBlockInvalid:1, Reserved
+ 0x5b, 0xe7, 0xc6, 0x86, 0xFE, 0xFF, 0xFF, 0xFF,
+ # WriteQueueSize: UINT64
+ 0xE0, 0xFF, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00
+}
+
+$(FTW_SPARE_BASE)|$(FTW_SPARE_SIZE)
+gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageFtwSpareBase64|gEfiMdeModulePkgTokenSpaceGuid.PcdFlashNvStorageFtwSpareSize
+#NV_FTW_SPARE
--
2.17.1

Please make the subject contain information.

/
Leif

On Fri, Mar 20, 2020 at 20:05:39 +0530, Pankaj Bansal wrote:

From: Pankaj Bansal <pankaj.bansal@...>

SocInit can be defined in SocLib.h
No need to make it extern in ArmPlatformLib

Signed-off-by: Pankaj Bansal <pankaj.bansal@...>

Reviewed-by: Leif Lindholm <leif@...>

But do please move it as early as possible in the series.

---
.../LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.c | 2 --
Silicon/NXP/Include/Library/SocLib.h | 8 ++++++++
2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.c b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.c
index 1e2e85f87dfe..dc81e7ba3101 100644
--- a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.c
+++ b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.c
@@ -16,8 +16,6 @@
#include <Ppi/ArmMpCoreInfo.h>
#include <Ppi/NxpPlatformGetClock.h>

-extern VOID SocInit (VOID);
-
/**
Get the clocks supplied by Platform(Board) to NXP Layerscape SOC IPs

diff --git a/Silicon/NXP/Include/Library/SocLib.h b/Silicon/NXP/Include/Library/SocLib.h
index 749aa230dec5..0ca68602618d 100644
--- a/Silicon/NXP/Include/Library/SocLib.h
+++ b/Silicon/NXP/Include/Library/SocLib.h
@@ -41,4 +41,12 @@ SocGetClock (
IN VA_LIST Args
);

+/**
+ Function to initialize SoC specific constructs
+ **/
+VOID
+SocInit (
+ VOID
+ );
+
#endif // SOC_LIB_H__
--
2.17.1

On Fri, Mar 20, 2020 at 20:05:38 +0530, Pankaj Bansal wrote:

From: Pankaj Bansal <pankaj.bansal@...>

Allocate Pages may allocate more memory than required for
VirtualMemoryTable.
There is no special requirement that VirtualMemoryTable size should be
page size aligned.

Therefore, replace AllocatePages with AllocatePool.

Signed-off-by: Pankaj Bansal <pankaj.bansal@...>

I don't object to this as such (although one comment), but what is the
purpose of this change?

My comment is that most other platforms use AllocatePages for this. So
this is diverging from the norm. Secondly, while I don't necessarily
*like* the current design (copied across most ARM platforms), it's
somewhat mitigated by the AllocatePages giving you a minimum of 128
entries before you start overwriting things. I don't know what you'll
overwrite if you do go too far, but you will overwrite it *before* the
ASSERT ever gets evaluated.

/
Leif

---
.../LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.inf | 1 +
.../LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLibMem.c | 5 +++--
2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.inf b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.inf
index 1faf99b99c54..c64032f32772 100644
--- a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.inf
+++ b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLib.inf
@@ -25,6 +25,7 @@ [Packages]

[LibraryClasses]
ArmLib
+ DebugLib
SocLib

[Sources.common]
diff --git a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLibMem.c b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLibMem.c
index f5fa308551aa..f8dd642e3cff 100644
--- a/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLibMem.c
+++ b/Platform/NXP/LS1043aRdbPkg/Library/ArmPlatformLib/ArmPlatformLibMem.c
@@ -43,10 +43,11 @@ ArmPlatformGetVirtualMemoryMap (

ASSERT (VirtualMemoryMap != NULL);

- VirtualMemoryTable = (ARM_MEMORY_REGION_DESCRIPTOR*)AllocatePages (
- EFI_SIZE_TO_PAGES (sizeof (ARM_MEMORY_REGION_DESCRIPTOR) * MAX_VIRTUAL_MEMORY_MAP_DESCRIPTORS));
+ VirtualMemoryTable = AllocatePool (sizeof (ARM_MEMORY_REGION_DESCRIPTOR) *
+ MAX_VIRTUAL_MEMORY_MAP_DESCRIPTORS);

if (VirtualMemoryTable == NULL) {
+ DEBUG ((DEBUG_ERROR, "%a: Error: Failed AllocatePool()\n", __FUNCTION__));
return;
}

--
2.17.1

On Mon, 30 Mar 2020 at 19:57, Ard Biesheuvel <ard.biesheuvel@...> wrote:

(adding Jian and Hao)

Thanks for the acks, and apologies for failing to cc the MdeModulePkg
maintainers.

Jian, Hao, do you have any opinion on this series?

Jian, Hao, Liming, Michael,

It is not always clear to me how you at Intel divide up the maintainer
duties between yourselves.

Liming and Jiewen have acked this entire series, but they are not the
MdeModulePkg maintainers, so I assume either Jian or Hao should
approve it as well before i can proceed with it.

If this is the case, could Jian and Hao please acknowledge that they
read this email, and perhaps indicate a timeframe within which they
will be able to give their verdict on these changes?

Thanks,
Ard.

On Mon, 30 Mar 2020 at 15:42, Gao, Liming <liming.gao@...> wrote:

Ack-by: Liming Gao <liming.gao@...>

-----Original Message-----
From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Yao, Jiewen
Sent: Friday, March 27, 2020 1:01 PM
To: Ard Biesheuvel <ard.biesheuvel@...>; devel@edk2.groups.io
Cc: Laszlo Ersek <lersek@...>; Leif Lindholm <leif@...>; Kinney, Michael D <michael.d.kinney@...>; Ni, Ray
<ray.ni@...>; Bret Barkelew <Bret.Barkelew@...>
Subject: Re: [edk2-devel] [PATCH 0/4] remove generation of EFI properties table

Acked-by: Jiewen Yao <Jiewen.yao@...>

I cannot remember if there is windows OS still using the properties table.
Maybe Microsoft people can comment.

If no, I agree we can remove the old code.

-----Original Message-----
From: Ard Biesheuvel <ard.biesheuvel@...>
Sent: Thursday, March 26, 2020 6:25 PM
To: devel@edk2.groups.io
Cc: Ard Biesheuvel <ard.biesheuvel@...>; Laszlo Ersek
<lersek@...>; Leif Lindholm <leif@...>; Kinney, Michael D
<michael.d.kinney@...>; Ni, Ray <ray.ni@...>; Yao, Jiewen
<jiewen.yao@...>; Bret Barkelew <Bret.Barkelew@...>
Subject: [PATCH 0/4] remove generation of EFI properties table

The EFI properties table is broken by design, deprecated, and seems to be
causing confusion as it is unclear to some how it differs from the memory
attributes table (which supersedes it). So let's get rid of the code that
generates it entirely, along with the GUID definitions, PCDs etc.

Due to how the two implementations are intertwined, patch #2 makes the
minimal changes required to stop producing the table (and to allow patch
#3 to remove the associated definitions from MdePkg). Patch #4 is optional
and merges the code together.

Cc: Laszlo Ersek <lersek@...>
Cc: Leif Lindholm <leif@...>
Cc: Michael D Kinney <michael.d.kinney@...>
Cc: Ray Ni <ray.ni@...>
Cc: Jiewen Yao <jiewen.yao@...>
Cc: Bret Barkelew <Bret.Barkelew@...>

Link: https://bugzilla.tianocore.org/show_bug.cgi?id=2633

Ard Biesheuvel (4):
OvmfPkg: remove handling of properties table
MdeModulePkg: disable properties table generation but retain the code
MdePkg: remove PropertiesTable GUID
MdeModulePkg/DxeCore: merge properties table routines into MAT
handling

MdeModulePkg/Core/Dxe/DxeMain.h | 9 -
MdeModulePkg/Core/Dxe/DxeMain.inf | 3 -
MdeModulePkg/Core/Dxe/DxeMain/DxeMain.c | 1 -
.../Core/Dxe/Misc/MemoryAttributesTable.c | 1226 ++++++++++++++-
MdeModulePkg/Core/Dxe/Misc/MemoryProtection.c | 1 -
MdeModulePkg/Core/Dxe/Misc/PropertiesTable.c | 1373 -----------------
MdeModulePkg/MdeModulePkg.dec | 24 -
MdeModulePkg/MdeModulePkg.uni | 21 -
MdePkg/Include/Guid/PropertiesTable.h | 31 -
MdePkg/MdePkg.dec | 3 -
OvmfPkg/OvmfPkgIa32.dsc | 1 -
OvmfPkg/OvmfPkgIa32X64.dsc | 1 -
OvmfPkg/OvmfPkgX64.dsc | 1 -
OvmfPkg/OvmfXen.dsc | 1 -
OvmfPkg/PlatformPei/Platform.c | 1 -
OvmfPkg/PlatformPei/PlatformPei.inf | 1 -
16 files changed, 1222 insertions(+), 1476 deletions(-)
delete mode 100644 MdeModulePkg/Core/Dxe/Misc/PropertiesTable.c
delete mode 100644 MdePkg/Include/Guid/PropertiesTable.h

--
2.17.1

Hi Ard,

I think adding a dependency in the module entry point
libs is also good way to guarantee an intrinsic lib
is available. I agree that it can provide module type
and arch specific mappings. The NULL lib instance can
do the same if the NULL instance is listed in the
module/arch specific library classes sections. a few
example section names.

[LibraryClasses.ARM.PEIM, LibraryClasses.AARCH64.PEIM]

[LibraryClasses.IA32.UEFI_DRIVER]

[LibraryClasses.X64.DXE_DRIVER]

So either way, the DSC files need to provide the
library mapping. The only difference between these
2 approaches is that adding a dependency to the
module entry point libs uses a formally defined
library class name for the intrinsic functions vs.
the un-named NULL library instance. A formally
defined library class name is better supported for
things like unit tests from the UnitTestFrameworkPkg.

The fltused issue would not go away of we removed the
float usage for OpenSSL. One or more other libs or the
module could use float/double types and this issue
would pop up again. I do agree it would be cleaner
if we could use OpenSSL without floating point.

I think adding an intrinsic lib for IA32/X64 for VS20xx
generation of intrinsic functions would address fltused
and other common C implementation styles that generate
intrinsic functions (e.g. 64-bit int math on 32-bit,
structure variable assignments, and loops that fill a buffer
with a const value) by VS20xx compilers. An intrinsic
lib for GCC would also help with these same common C
implementation styles that generate intrinsic functions.

Mike